Snapshot 268814
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"incidents": [
{
"created_at": "2026-11-13T01:26:31Z",
"id": "01M7986VDW8YZ7JCTQX5C0ZA5E",
"impact": "minor",
"incident_updates": [
{
"body": "The incident has been resolved, operations are back to normal.",
"created_at": "2026-11-13T12:26:30Z",
"display_at": "2026-11-13T12:26:30Z",
"id": "01M7ADZB3C4VG3FJDPYSPC9V21",
"incident_id": "01M7986VDW8YZ7JCTQX5C0ZA5E",
"status": "resolved",
"updated_at": "2026-11-13T12:26:30Z"
},
{
"body": "We have applied mitigation for the performance degradation affecting time series metadata APIs in `az-phx-001`. The impact has been mitigated, and we are monitoring the service to confirm continued stability.",
"created_at": "2026-11-13T01:32:26Z",
"display_at": "2026-11-13T00:00:00Z",
"id": "01M798HPJDCG4WBZGDFXZM6W5V",
"incident_id": "01M7986VDW8YZ7JCTQX5C0ZA5E",
"status": "monitoring",
"updated_at": "2026-11-13T01:32:51Z"
},
{
"body": "We are applying mitigation to address performance degradation affecting time series metadata APIs in `az-phx-001`. We will monitor the results and provide an update once the mitigation has been verified.",
"created_at": "2026-11-13T01:30:15Z",
"display_at": "2026-11-12T22:44:00Z",
"id": "01M798DPT0G6BTER7ZK536MDMA",
"incident_id": "01M7986VDW8YZ7JCTQX5C0ZA5E",
"status": "identified",
"updated_at": "2026-11-13T01:30:40Z"
},
{
"body": "We are investigating performance degradation affecting time series metadata list, search, and aggregate APIs in `az-phx-001`. Requests may intermittently fail or return stale results. We will provide an update as more information becomes available.",
"created_at": "2026-11-13T01:26:31Z",
"display_at": "2026-11-12T22:00:00Z",
"id": "01M7986VDWHTPNG21QCGX26P2W",
"incident_id": "01M7986VDW8YZ7JCTQX5C0ZA5E",
"status": "investigating",
"updated_at": "2026-11-13T01:27:06Z"
}
],
"name": "Time series metadata API performance degradation in az-phx-001",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-11-13T12:26:30Z",
"status": "resolved",
"updated_at": "2026-11-13T12:26:31Z"
},
{
"created_at": "2026-11-12T07:12:40Z",
"id": "01M779KYVCK0803YTFMD5FF8RH",
"impact": "minor",
"incident_updates": [
{
"body": "This incident has been resolved. The Data Modeling API in the az-phx-001 cluster is operating normally. All affected services are functioning as expected.",
"created_at": "2026-11-12T12:16:54Z",
"display_at": "2026-11-12T12:16:54Z",
"id": "01M77V117NZYHJKKNCT26AT7S3",
"incident_id": "01M779KYVCK0803YTFMD5FF8RH",
"status": "resolved",
"updated_at": "2026-11-12T12:16:54Z"
},
{
"body": "We have applied a fix for the issue affecting Data Modeling requests in the `az-phx-001`cluster. Error rates have returned to near zero, and service operation appears back to normal. We are monitoring to confirm continued stability.",
"created_at": "2026-11-12T07:23:33Z",
"display_at": "2026-11-12T07:19:00Z",
"id": "01M77A7X4WHRZ5AJ6V0PXCF3DQ",
"incident_id": "01M779KYVCK0803YTFMD5FF8RH",
"status": "monitoring",
"updated_at": "2026-11-12T07:24:34Z"
},
{
"body": "We are investigating elevated error rates affecting Data Modeling requests in the `az-phx-001` cluster. The service remains available, but some requests may return HTTP 500 errors. Our engineering team is working to restore normal operation. We will provide further updates as more information becomes available.",
"created_at": "2026-11-12T07:12:40Z",
"display_at": "2026-11-12T06:53:00Z",
"id": "01M779KYVC393WGQ3GAMRR5G6A",
"incident_id": "01M779KYVCK0803YTFMD5FF8RH",
"status": "investigating",
"updated_at": "2026-11-12T07:13:44Z"
}
],
"name": "Elevated errors affecting Data Modeling in az-phx-001",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-11-12T12:16:54Z",
"status": "resolved",
"updated_at": "2026-11-12T12:16:54Z"
},
{
"created_at": "2026-11-03T18:06:22Z",
"id": "01M6H9EEK3TG93H95CGSFY1V7F",
"impact": "minor",
"incident_updates": [
{
"body": "This incident has been resolved. The Time Series API in the Statnett cluster has been restored and is operating normally. All affected services are functioning as expected.",
"created_at": "2026-11-04T09:31:32Z",
"display_at": "2026-11-04T09:31:32Z",
"id": "01M6JYCG1E1ZH3Y2MXHKWRMB7N",
"incident_id": "01M6H9EEK3TG93H95CGSFY1V7F",
"status": "resolved",
"updated_at": "2026-11-04T09:31:32Z"
},
{
"body": "We are monitoring the Cognite Data Fusion API in the statnett cluster to ensure the service remains stable following the restoration.",
"created_at": "2026-11-04T05:14:26Z",
"display_at": "2026-11-03T18:33:00Z",
"id": "01M6JFNQBHQ465AS1HNSW5K1PT",
"incident_id": "01M6H9EEK3TG93H95CGSFY1V7F",
"status": "monitoring",
"updated_at": "2026-11-04T05:14:59Z"
},
{
"body": "We identified a certificate configuration issue that prevented the Time Series API in the statnett cluster from connecting to an internal caching service. This caused an outage for time-series metadata and subscription reads, and partially affected datapoint ingestion and reads. Service has been restored by correcting the certificate configuration",
"created_at": "2026-11-04T05:09:43Z",
"display_at": "2026-11-03T18:09:00Z",
"id": "01M6JFD3AMJXX3FT2VHB5BZMFB",
"incident_id": "01M6H9EEK3TG93H95CGSFY1V7F",
"status": "identified",
"updated_at": "2026-11-04T05:10:21Z"
},
{
"body": "We are currently investigating an issue impacting the Time Series API in the statnett cluster.\n\n**Current Impact:**\n\n• **Timeseries Metadata Read** and **Subscription Read** are currently experiencing a total outage.\n\n• **Datapoint ingestion** and **Datapoint read** are partially impacted with elevated error rates and latency\n\nOur engineering team has identified a certificate validation issue connecting to internal caching services and is actively working on refreshing credentials to restore affected pods and services. We will provide further updates as progress is made.",
"created_at": "2026-11-03T18:06:22Z",
"display_at": "2026-11-03T17:06:00Z",
"id": "01M6H9EEK3TS60WPA8VRBJ7JJB",
"incident_id": "01M6H9EEK3TG93H95CGSFY1V7F",
"status": "investigating",
"updated_at": "2026-11-04T05:12:36Z"
}
],
"name": "Service Degradation for Time Series API in statnett Cluster",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-11-04T09:31:32Z",
"status": "resolved",
"updated_at": "2026-11-04T09:31:32Z"
},
{
"created_at": "2026-10-30T07:15:29Z",
"id": "01M65TKRZ2F7PE80ZRE0SCQC8Q",
"impact": "minor",
"incident_updates": [
{
"body": "Files uploaded via uploadUrl in az-tyo-gp-001 are now available as expected, including in InField.",
"created_at": "2026-10-30T15:02:52Z",
"display_at": "2026-10-30T15:02:52Z",
"id": "01M66NBJYT10E53ZD93CBZD8M5",
"incident_id": "01M65TKRZ2F7PE80ZRE0SCQC8Q",
"status": "resolved",
"updated_at": "2026-10-30T15:02:52Z"
},
{
"body": "**Monitoring:** We've applied a fix for the issue where files uploaded via uploadUrl weren't showing as uploaded and couldn't be accessed. File upload processing has recovered in the affected clusters and customers have confirmed their files can be accessed again.\n\n**We are now monitoring** to make sure the remaining processing backlog clears and upload status updates normally in all clusters. We'll post another update once we've confirmed everything is back to normal.",
"created_at": "2026-10-30T09:58:55Z",
"display_at": "2026-10-30T09:58:55Z",
"id": "01M663Z1GGZ4MX925ZZD75N0EM",
"incident_id": "01M65TKRZ2F7PE80ZRE0SCQC8Q",
"status": "monitoring",
"updated_at": "2026-10-30T09:58:55Z"
},
{
"body": "Newly uploaded files in the Tokyo region (`az-tyo-gp-001`) may take longer than usual to become available. This affects downloads, viewing newly uploaded photos in InField, and workflows that depend on upload completion. Downloads and workflows that wait for a file to finish uploading are affected. File contents already in storage are unchanged. \n\nWe have identified an issue with upload-status processing and are working on a fix.",
"created_at": "2026-10-30T07:21:14Z",
"display_at": "2026-10-30T07:05:00Z",
"id": "01M65TYA9B4XAG8PTTRQ2T4JV5",
"incident_id": "01M65TKRZ2F7PE80ZRE0SCQC8Q",
"status": "identified",
"updated_at": "2026-10-30T07:21:31Z"
},
{
"body": "Newly uploaded files in the Tokyo region (`az-tyo-gp-001`) may remain marked as not uploaded and unavailable for longer than usual. This affects downloads, viewing newly uploaded photos in InField, and workflows that wait for upload completion. File contents already in storage are unchanged. We are continuing to investigate the full scope of the impact.",
"created_at": "2026-10-30T07:15:29Z",
"display_at": "2026-10-30T05:56:00Z",
"id": "01M65TKRZ26KM2D8V89ZARVC66",
"incident_id": "01M65TKRZ2F7PE80ZRE0SCQC8Q",
"status": "investigating",
"updated_at": "2026-10-30T07:16:35Z"
}
],
"name": "uploaded-files-via-uploadurl-do-not-appear-in-az-tyo-gp",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-10-30T15:02:52Z",
"status": "resolved",
"updated_at": "2026-10-30T15:02:52Z"
},
{
"created_at": "2026-10-16T12:14:00Z",
"id": "01M52A4AR15SYYPT6CRK6SRPHD",
"impact": "major",
"incident_updates": [
{
"body": "The issue affecting 3D services in `az-tyo-002` has been fixed by increasing the health probe startup delay. Service has been restored, and is now resolved. ",
"created_at": "2026-10-16T14:06:34Z",
"display_at": "2026-10-16T14:06:34Z",
"id": "01M52GJEBJQX782AEZCY0DZ74S",
"incident_id": "01M52A4AR15SYYPT6CRK6SRPHD",
"status": "resolved",
"updated_at": "2026-10-16T14:06:34Z"
},
{
"body": "The fix increased the health probe startup delay to 60 seconds. The 3D service in `az-tyo-002` is back online, and pods appear to have stopped restarting. The engineering team is now **Monitoring** and checking the stability while awaiting customer confirmation.",
"created_at": "2026-10-16T12:45:59Z",
"display_at": "2026-10-16T12:34:00Z",
"id": "01M52BYVZCS1D088X1KRJJH6M6",
"incident_id": "01M52A4AR15SYYPT6CRK6SRPHD",
"status": "monitoring",
"updated_at": "2026-10-16T12:46:18Z"
},
{
"body": "The **3D service** in the **az-tyo-002** cluster is experiencing service disruptions due to repeated service restarts during startup health checks**.** \n\nThe engineering team has identified the mitigation and is currently deploying an adjustment to the startup probe delays to stabilize the pods and restore service availability",
"created_at": "2026-10-16T12:18:40Z",
"display_at": "2026-10-16T12:05:00Z",
"id": "01M52ACVQ0HZDSR91RRZWZMF74",
"incident_id": "01M52A4AR15SYYPT6CRK6SRPHD",
"status": "identified",
"updated_at": "2026-10-16T12:20:30Z"
},
{
"body": "We are investigating reports of service degradation and disruption affecting the **3D service** in the **az-tyo-002** cluster. 3D models and related API endpoints may be unavailable. Our engineering team is currently investigating the issue",
"created_at": "2026-10-16T12:14:01Z",
"display_at": "2026-10-16T11:50:00Z",
"id": "01M52A4AR11PESX31C2KM588YN",
"incident_id": "01M52A4AR15SYYPT6CRK6SRPHD",
"status": "investigating",
"updated_at": "2026-10-16T12:19:50Z"
}
],
"name": "3D Service Outage in az-tyo-002",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-10-16T14:06:34Z",
"status": "resolved",
"updated_at": "2026-10-16T14:06:34Z"
},
{
"created_at": "2026-10-14T15:56:34Z",
"id": "01M4XJ2D0SQNZM7BN77ZJNK3DT",
"impact": "major",
"incident_updates": [
{
"body": "Upstream provider Microsoft has confirmed that the underlying service-side timeout issues in Azure OpenAI (East US 2) are fully resolved.\n\nCustomer-facing mitigation remains in place and all Atlas AI agent chat metrics, traffic and availability have returned to normal operating baselines across all clusters. ",
"created_at": "2026-10-14T23:39:12Z",
"display_at": "2026-10-14T23:39:12Z",
"id": "01M4YCHGPW3R4AHZATZHDQ3RA0",
"incident_id": "01M4XJ2D0SQNZM7BN77ZJNK3DT",
"status": "resolved",
"updated_at": "2026-10-14T23:39:12Z"
},
{
"body": "We are investigating elevated errors affecting Atlas AI agent chat across multiple regions. Users may experience failed chat requests. Initial findings suggest upstream timeouts from Azure OpenAI in the East US 2 region may be contributing to the issue, including requests originating from other regions. We are continuing to investigate the cause and impact.",
"created_at": "2026-10-14T15:56:34Z",
"display_at": "2026-10-14T15:56:34Z",
"id": "01M4XJ2D0S022Y2N38AZX49D0M",
"incident_id": "01M4XJ2D0SQNZM7BN77ZJNK3DT",
"status": "investigating",
"updated_at": "2026-10-14T15:56:34Z"
}
],
"name": "Elevated errors in Atlas AI agent chat",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-10-14T23:39:12Z",
"status": "resolved",
"updated_at": "2026-10-14T23:39:12Z"
},
{
"created_at": "2026-10-08T12:43:11Z",
"id": "01M4DRKZSC1P9XANC3YYQQXX6B",
"impact": "major",
"incident_updates": [
{
"body": "**All services have recovered and are operating normally.** ",
"created_at": "2026-10-08T12:46:06Z",
"display_at": "2026-10-08T12:06:00Z",
"id": "01M4DRSBAEH1RNJQJQ4P91N29T",
"incident_id": "01M4DRKZSC1P9XANC3YYQQXX6B",
"status": "resolved",
"updated_at": "2026-10-08T12:46:22Z"
},
{
"body": "**Our engineering team identified the regression and completed a rollback to restore service. All services have recovered and are operating normally. We are actively monitoring the systems to ensure ongoing stability**",
"created_at": "2026-10-08T12:44:07Z",
"display_at": "2026-10-08T12:04:00Z",
"id": "01M4DRNPYJ6C28TNQBKPSR3DS6",
"incident_id": "01M4DRKZSC1P9XANC3YYQQXX6B",
"status": "monitoring",
"updated_at": "2026-10-08T12:45:52Z"
},
{
"body": "**We experienced an issue with the Time Series API that caused temporary service disruptions across select clusters:**\n\n• **az-arn-001:** Time series datapoint ingestion (ingesting and deleting data points) experienced HTTP 503 errors and unavailability between 13:49 and 14:02 CEST.\n\n• **az-sin-sp-001:** Time series metadata endpoints (listing, searching, and creating classic time series) experienced a brief period of errors between 14:02 and 14:05 CEST. Datapoint ingestion remained operational.",
"created_at": "2026-10-08T12:43:11Z",
"display_at": "2026-10-08T11:49:00Z",
"id": "01M4DRKZSC3771FTVTP5N7X9R9",
"incident_id": "01M4DRKZSC1P9XANC3YYQQXX6B",
"status": "identified",
"updated_at": "2026-10-08T12:43:30Z"
}
],
"name": "Time Series Service Disruption in az-arn-001 and az-sin-sp-001",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-10-08T12:06:00Z",
"status": "resolved",
"updated_at": "2026-10-08T12:46:06Z"
},
{
"created_at": "2026-10-02T13:54:11Z",
"id": "01M3YE9PP87BEPTH395YW2S8XF",
"impact": "minor",
"incident_updates": [
{
"body": "The workflow/transformation performance issue has been resolved. \nIncreasing the project’s task-pool priority and unscheduling bootcamp transformations significantly improved performance, reducing average transformation start delays.",
"created_at": "2026-10-08T11:24:17Z",
"display_at": "2026-10-07T09:53:00Z",
"id": "01M4DM3H6PC7AGNDPRF1KS5EET",
"incident_id": "01M3YE9PP87BEPTH395YW2S8XF",
"status": "resolved",
"updated_at": "2026-10-08T11:26:01Z"
},
{
"body": "The issue is fixed. We are continuing to monitor this.",
"created_at": "2026-10-06T15:40:52Z",
"display_at": "2026-10-06T15:40:52Z",
"id": "01M48XZX9QP2PJ8ZWJC8QKWWK4",
"incident_id": "01M3YE9PP87BEPTH395YW2S8XF",
"status": "monitoring",
"updated_at": "2026-10-06T15:40:52Z"
},
{
"body": "We are currently investigating an issue causing degraded performance for Transformations and Workflows in the **westeurope-1** cluster.\n\n**Customer Impact:**\n\n• Delayed start times for scheduled transformation jobs.\n\n• Extended overall runtime for workflows and chained data pipelines.\n\n• Jobs are continuing to process and complete, but latency is significantly higher than baseline.\n\n**Current Status \u0026 Actions:**\nEngineering team is actively investigating resource contention and applying mitigations across the worker pools to restore normal throughput.\n\nWe will keep you posted on the progress.",
"created_at": "2026-10-02T13:54:11Z",
"display_at": "2026-10-02T13:54:11Z",
"id": "01M3YE9PP8EGNF9K3GDNA5WW39",
"incident_id": "01M3YE9PP87BEPTH395YW2S8XF",
"status": "investigating",
"updated_at": "2026-10-02T14:46:12Z"
}
],
"name": "Degraded Transformation \u0026 Workflow Performance in westeurope-1",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-10-07T09:53:00Z",
"status": "resolved",
"updated_at": "2026-10-08T11:24:17Z"
},
{
"created_at": "2026-10-02T14:15:48Z",
"id": "01M3YFH8VVWJTBDRQK3SPXBHTT",
"impact": "minor",
"incident_updates": [
{
"body": "**The issue causing increased latency and degraded performance in the** `az-phx-001` **cluster has been resolved. All affected services and API response times are back to normal operation**",
"created_at": "2026-10-06T23:47:57Z",
"display_at": "2026-10-06T17:33:00Z",
"id": "01M49SVSE1RJ40WCE74P2EW7FA",
"incident_id": "01M3YFH8VVWJTBDRQK3SPXBHTT",
"status": "resolved",
"updated_at": "2026-10-06T23:48:24Z"
},
{
"body": "**A fix has been deployed by migrating the affected database storage pods to a new node pool in the** `az-phx-001` **cluster. Cluster latency and performance have returned to normal levels, and our team is actively monitoring the service to ensure stability.**",
"created_at": "2026-10-06T23:45:59Z",
"display_at": "2026-10-06T14:48:00Z",
"id": "01M49SR5SB7YWQWB06EWHZZG37",
"incident_id": "01M3YFH8VVWJTBDRQK3SPXBHTT",
"status": "monitoring",
"updated_at": "2026-10-06T23:46:55Z"
},
{
"body": "We are currently investigating reports of increased latency and degraded performance affecting services in the az-phx-001 cluster (Azure West US 3). \n\nOur engineering teams are actively investigating the underlying storage performance and working on mitigating the issue. \n\nNext update will be provided within as soon as more information becomes available.",
"created_at": "2026-10-02T14:15:48Z",
"display_at": "2026-10-02T14:15:48Z",
"id": "01M3YFH8VVFKAX219G1T30F9VX",
"incident_id": "01M3YFH8VVWJTBDRQK3SPXBHTT",
"status": "investigating",
"updated_at": "2026-10-02T14:15:48Z"
}
],
"name": "\t Performance Degradation in az-phx-001 (US West)",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-10-06T17:33:00Z",
"status": "resolved",
"updated_at": "2026-10-06T23:47:57Z"
},
{
"created_at": "2026-09-11T02:44:36Z",
"id": "01M275MJA0QEH7HH6DE0Z740GW",
"impact": "minor",
"incident_updates": [
{
"body": "This incident has been resolved. Data modeling services in `az-tyo-gp-001` have fully recovered and error rates have returned to normal. All systems are operating as expected.\n",
"created_at": "2026-10-01T07:11:33Z",
"display_at": "2026-10-01T07:11:33Z",
"id": "01M3V4VQ638EQTW7FS2XJKERRG",
"incident_id": "01M275MJA0QEH7HH6DE0Z740GW",
"status": "resolved",
"updated_at": "2026-10-01T07:11:33Z"
},
{
"body": "**The engineering team has allocated additional memory resources to the affected services, and normal performance has been restored with error rates dropping to zero. We are continuing to closely monitor the system to ensure ongoing stability. Further updates will be provided once the incident is fully resolved.**",
"created_at": "2026-09-11T03:09:14Z",
"display_at": "2026-09-11T03:09:14Z",
"id": "01M2771MSQA4JRCGQ10MM6ZCCN",
"incident_id": "01M275MJA0QEH7HH6DE0Z740GW",
"status": "monitoring",
"updated_at": "2026-09-11T03:09:14Z"
},
{
"body": "**We are currently investigating an issue causing elevated 503 error rates and service degradation for Data Modeling services within the az-tyo-gp-001 cluster. Users and applications in this cluster may experience intermittent errors or failures when accessing data modeling endpoints and related features.**\n\n**Our engineering team is actively investigating the root cause to restore normal operations. Further updates will be shared as more information becomes available.**",
"created_at": "2026-09-11T02:44:36Z",
"display_at": "2026-09-11T02:44:36Z",
"id": "01M275MJA0NR18943NA1JPD9RE",
"incident_id": "01M275MJA0QEH7HH6DE0Z740GW",
"status": "investigating",
"updated_at": "2026-09-11T02:44:36Z"
}
],
"name": "Data Modeling service degradation in az-tyo-gp-001",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-10-01T07:11:33Z",
"status": "resolved",
"updated_at": "2026-10-01T07:11:33Z"
},
{
"created_at": "2026-09-29T15:52:20Z",
"id": "01M3PXVWQQQWSRKE7429SY5D96",
"impact": "minor",
"incident_updates": [
{
"body": "The issue causing slow response times for Azure OpenAI models in the EU Datazone has been resolved.",
"created_at": "2026-09-30T16:16:35Z",
"display_at": "2026-09-30T16:16:35Z",
"id": "01M3SHN01SPC8M98PFS4VMAHB6",
"incident_id": "01M3PXVWQQQWSRKE7429SY5D96",
"status": "resolved",
"updated_at": "2026-09-30T16:16:35Z"
},
{
"body": "Microsoft Azure has reported that the underlying platform issue in the `swedencentral` region has now been mitigated.\n\nOur internal metrics indicate that service health and model response times are recovering. We are actively observing system performance and endpoint latencies across all affected clusters to verify sustained stability.\n\nFurther updates will be shared as soon as full recovery is confirmed.",
"created_at": "2026-09-29T19:37:14Z",
"display_at": "2026-09-29T19:00:00Z",
"id": "01M3QAQNQ02XEQEDE0ZGYK0TKY",
"incident_id": "01M3PXVWQQQWSRKE7429SY5D96",
"status": "monitoring",
"updated_at": "2026-09-29T19:38:07Z"
},
{
"body": "We are currently tracking an external incident with Microsoft Azure affecting the Azure OpenAI Service in `swedencentral`, which impacts models deployed in the EU Datazone.\nDue to this underlying third-party provider issue outside of Cognite’s direct control, customers in the affected clusters may experience slower response times, increased LLM latencies, or intermittent request failures when using:\n\n• Atlas AI agents\n\n• Chat completion requests\n\n• Document summary features in Canvas\n\n\nIn some cases, switching to a non-OpenAI model may help mitigate the issue, however, tool calls may still require interaction with an OpenAI model and could continue to experience delays.\n\nOur team is actively monitoring the situation alongside Microsoft’s response. Further updates will be shared as soon as Azure services are restored. You can also view the direct status of the underlying Azure platform at \u003chttps://azure.status.microsoft/en-us/status\u003e",
"created_at": "2026-09-29T15:52:21Z",
"display_at": "2026-09-29T15:52:20Z",
"id": "01M3PXVWQQ1DJC7M2TZK8STN8C",
"incident_id": "01M3PXVWQQQWSRKE7429SY5D96",
"status": "investigating",
"updated_at": "2026-09-29T16:07:45Z"
}
],
"name": "Slow response time on OpenAI Azure models EU",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-30T16:16:35Z",
"status": "resolved",
"updated_at": "2026-09-30T16:16:35Z"
},
{
"created_at": "2026-09-28T11:18:10Z",
"id": "01M3KVS5312YZX9F9YHSPT24KD",
"impact": "major",
"incident_updates": [
{
"body": "All scheduled calculations have resumed and data point processing is fully operational. We have verified full recovery and confirmed normal service performance.\n",
"created_at": "2026-09-30T11:15:03Z",
"display_at": "2026-09-30T11:15:03Z",
"id": "01M3S0CW2GZQ4E6D1XCH1PNYXD",
"incident_id": "01M3KVS5312YZX9F9YHSPT24KD",
"status": "resolved",
"updated_at": "2026-09-30T11:15:03Z"
},
{
"body": "**_Monitoring_** _– Scheduled calculations have resumed and data points are updating as expected. Our team is actively monitoring the service and verifying data flow stability with affected users before marking the incident as fully resolved._\n",
"created_at": "2026-09-30T09:47:24Z",
"display_at": "2026-09-30T09:47:24Z",
"id": "01M3RVCCN9HQ6DRV7R3N3037Z1",
"incident_id": "01M3KVS5312YZX9F9YHSPT24KD",
"status": "monitoring",
"updated_at": "2026-09-30T09:47:24Z"
},
{
"body": "**Monitoring** - Fixes to scale worker capacity and improve service autoscaling have been deployed. We are observing scheduled calculations resuming execution. Due to the backlog accumulated during the degradation, jobs are currently catching up and may experience processing delays over the next few hours. We are actively monitoring the service until all scheduled calculations are fully up to date.",
"created_at": "2026-09-28T15:57:16Z",
"display_at": "2026-09-28T15:57:16Z",
"id": "01M3MBR6AMGZC7Y2AY20MERED5",
"incident_id": "01M3KVS5312YZX9F9YHSPT24KD",
"status": "monitoring",
"updated_at": "2026-09-28T15:57:16Z"
},
{
"body": "**Current Status Update — INC-3602 (Scheduled Calculations in Charts)**\n\n**Impact:**\nScheduled calculation executions in Charts are failing or significantly delayed, preventing new data points from being written. The issue currently appears isolated to cluster `az-tyo-gp-001` (verified working normally on `aw-tyo-001`).\n\n**Current Findings:**\n\n• High CPU utilization (100–1000%) observed on backend workers, causing jobs to fall hours behind schedule.\n\n• Extensive timeout errors and ~236 failed jobs recorded across the cluster over the weekend.\n\n• Newly created and existing scheduled calculations are affected.\n\n**Current Actions \u0026 Next Steps:**\n\n• Engineering is investigating the root cause of the CPU spike and timeout failures.\n\n• Evaluating remediation options to relieve backend load and process the backlog of calculations.\n\n ",
"created_at": "2026-09-28T12:28:19Z",
"display_at": "2026-09-28T12:28:19Z",
"id": "01M3KZSK9PEMP5WXMZMKVY43M5",
"incident_id": "01M3KVS5312YZX9F9YHSPT24KD",
"status": "investigating",
"updated_at": "2026-09-28T12:28:19Z"
},
{
"body": "_We are currently investigating an issue where scheduled calculations created in Charts are delayed or failing to write new data points. Our engineering team is actively investigating the root cause. Further updates will be provided as soon as they become available._",
"created_at": "2026-09-28T11:18:10Z",
"display_at": "2026-09-28T11:18:10Z",
"id": "01M3KVS531AB0ZR3QJ98TZT2FS",
"incident_id": "01M3KVS5312YZX9F9YHSPT24KD",
"status": "investigating",
"updated_at": "2026-09-28T11:18:10Z"
}
],
"name": "Degraded Performance - Scheduled Calculations in Charts",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-30T11:15:03Z",
"status": "resolved",
"updated_at": "2026-09-30T11:15:03Z"
},
{
"created_at": "2026-09-22T10:08:52Z",
"id": "01M349DY9VV44Z7MDRJGGCTYJ7",
"impact": "minor",
"incident_updates": [
{
"body": "The performance degradation affecting Data Modeling search and indexing in **westeurope-1** has been fully resolved.\n\nSyncer capacity has been restored, cluster health metrics have returned to normal, and all indexing backlogs have cleared. Search, aggregation, and data modeling queries are functioning as expected.\n\nWe are continuing to monitor the services to ensure ongoing stability.\n\n\n",
"created_at": "2026-09-24T12:16:35Z",
"display_at": "2026-09-24T12:16:35Z",
"id": "01M39NH7HGP6W17825FQBF1QH7",
"incident_id": "01M349DY9VV44Z7MDRJGGCTYJ7",
"status": "resolved",
"updated_at": "2026-09-24T12:16:35Z"
},
{
"body": "The underlying cluster in `westeurope-1` has stabilized, and search and aggregate queries are functioning normally. Indexing pipelines are currently processing the remaining backlog, so some users may still see slight delays in newly ingested data appearing. We are monitoring the recovery closely.",
"created_at": "2026-09-22T10:31:57Z",
"display_at": "2026-09-22T10:31:57Z",
"id": "01M34AR6RPHDCBJ1SXWRWGM21Y",
"incident_id": "01M349DY9VV44Z7MDRJGGCTYJ7",
"status": "monitoring",
"updated_at": "2026-09-22T10:31:57Z"
},
{
"body": "We have identified an issue affecting search and data modeling indexing in the `westeurope-1` region. Users and automated pipelines in this cluster may experience delays in newly ingested data or schema updates appearing in search and data modeling views. Querying existing indexed data and other CDF services remain operational.\n\nOur engineering team has implemented traffic-shedding and resource adjustments to stabilize the underlying cluster, and indexing throughput is beginning to recover. We are closely monitoring cluster health and will provide our next update soon,",
"created_at": "2026-09-22T10:08:52Z",
"display_at": "2026-09-22T10:08:52Z",
"id": "01M349DY9V2RJZKCXF9FDHK4JZ",
"incident_id": "01M349DY9VV44Z7MDRJGGCTYJ7",
"status": "investigating",
"updated_at": "2026-09-22T10:08:52Z"
}
],
"name": " Degraded Data Modeling Indexing and Search in westeurope-1",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-24T12:16:35Z",
"status": "resolved",
"updated_at": "2026-09-24T12:16:35Z"
},
{
"created_at": "2026-09-23T17:45:19Z",
"id": "01M37NYE7ZM08MSQB1ST0KN815",
"impact": "minor",
"incident_updates": [
{
"body": "The performance degradation affecting Data Modeling search and indexing in **westeurope-1** has been fully resolved.\n\nSyncer capacity has been restored, cluster health metrics have returned to normal, and all indexing backlogs have cleared. Search, aggregation, and data modeling queries are functioning as expected.\n\nWe are continuing to monitor the services to ensure ongoing stability.",
"created_at": "2026-09-24T12:08:01Z",
"display_at": "2026-09-24T12:08:01Z",
"id": "01M39N1H5E3AZPT1CAMWC20D0J",
"incident_id": "01M37NYE7ZM08MSQB1ST0KN815",
"status": "resolved",
"updated_at": "2026-09-24T12:08:01Z"
},
{
"body": "• **Problem:** The Propertygraph Elasticsearch cluster in **westeurope-1** experienced degradation, temporarily entering yellow and red states.\n\n• **Impact:** Customers with projects in `westeurope-1` may experience delays in newly created instances appearing in search results, along with elevated latency for search and aggregate queries affecting data modeling indexing (`datamodelstorage`) Core data storage is unaffected.\n\n• **Root Cause:** A syncer update triggered concurrent restarts across syncer deployments simultaneously, overwhelming the search master nodes with high-volume reads. In addition, an search pod encountered a PVC mount failure during rolling updates.\n\n• **Status \u0026 Mitigation:** Mitigation steps have been applied. Our engineering team scaled down the syncers to allow the cluster to recover to a healthy green state, subsequently re-enabled the synchronizers in stages, and resolved the PVC volume mount issue. The cluster is stabilizing, and our team is actively monitoring system performance and recovery.",
"created_at": "2026-09-23T19:17:38Z",
"display_at": "2026-09-23T19:10:00Z",
"id": "01M37V7FCN1QB3ERRQR19XP56B",
"incident_id": "01M37NYE7ZM08MSQB1ST0KN815",
"status": "monitoring",
"updated_at": "2026-09-23T19:18:00Z"
},
{
"body": "We are currently experiencing performance degradation affecting search and data modeling indexing services in the **westeurope-1** region.\n**Impact:** Search and aggregate queries may experience elevated latency, and newly created instances or index updates may take longer to appear in search results. Core data storage is unaffected.\nOur engineering team is actively investigating the issue and working on mitigation steps to stabilize the service. We will provide updates as more information becomes available",
"created_at": "2026-09-23T17:45:19Z",
"display_at": "2026-09-23T17:45:19Z",
"id": "01M37NYE7ZTC0B4F1ESWK0VFE4",
"incident_id": "01M37NYE7ZM08MSQB1ST0KN815",
"status": "investigating",
"updated_at": "2026-09-23T17:45:19Z"
}
],
"name": "Degraded search and data modeling indexing performance",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-24T12:08:01Z",
"status": "resolved",
"updated_at": "2026-09-24T12:08:01Z"
},
{
"created_at": "2026-09-22T21:06:27Z",
"id": "01M35F20AB8EVKK1F303HZ26DK",
"impact": "minor",
"incident_updates": [
{
"body": "Service has been restored. The Elasticsearch cluster in westeurope-1 has stabilized, and search, aggregation, and data-modeling indexing are operating normally. ",
"created_at": "2026-09-23T09:05:28Z",
"display_at": "2026-09-23T09:05:28Z",
"id": "01M36R6HWZ5YQPGP8VD7ER2PYX",
"incident_id": "01M35F20AB8EVKK1F303HZ26DK",
"status": "resolved",
"updated_at": "2026-09-23T09:05:28Z"
},
{
"body": "The Elasticsearch cluster in **westeurope-1** became overloaded and entered a red state. This degraded search and data-modeling indexing. search and aggregate requests could be slow or fail, while new instances and index changes could take longer to appear.\n\n**Root cause:** A rollout restart of **pg3** overloaded the Elasticsearch cluster.\n\n**Fixing status:** The incident is now in **Monitoring**. Engineering team paused some background synchronizers to reduce load, allowed Elasticsearch to stabilize, and then started re-enabling the synchronizers. The fix was marked as applied.",
"created_at": "2026-09-23T00:07:29Z",
"display_at": "2026-09-22T22:43:00Z",
"id": "01M35SDFSKC1AA1PK25K3G6ER1",
"incident_id": "01M35F20AB8EVKK1F303HZ26DK",
"status": "monitoring",
"updated_at": "2026-09-23T00:08:09Z"
},
{
"body": "We are currently investigating performance degradation and high latency affecting search and data modeling services in the **westeurope-1** region.\n\n**Impact:** Search and aggregate requests may experience high latency or temporary failures. Newly created instances or index updates may take longer than expected to appear.\n\n**Status:** Our engineering team is currently scaling down background syncing processes to reduce load and restore cluster stability. Further updates will be posted as soon as they are available.",
"created_at": "2026-09-22T21:06:27Z",
"display_at": "2026-09-22T19:57:00Z",
"id": "01M35F20AB6SHYZD54JET8MFA8",
"incident_id": "01M35F20AB8EVKK1F303HZ26DK",
"status": "investigating",
"updated_at": "2026-09-22T21:10:57Z"
}
],
"name": "Indexing in data modeling is degraded",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-23T09:05:28Z",
"status": "resolved",
"updated_at": "2026-09-23T09:05:28Z"
},
{
"created_at": "2026-09-11T07:20:32Z",
"id": "01M27NDSVKCS0JGAMJHPHH6RCM",
"impact": "minor",
"incident_updates": [
{
"body": "Increased error rates were found to be confined to a particular client configuration with excessive retries, and were mitigated. Overall cluster availability remained nominal.",
"created_at": "2026-09-18T19:18:43Z",
"display_at": "2026-09-18T19:18:43Z",
"id": "01M2TZ9VK7ZXYXZZG1RWF6491D",
"incident_id": "01M27NDSVKCS0JGAMJHPHH6RCM",
"status": "resolved",
"updated_at": "2026-09-18T19:18:43Z"
},
{
"body": "**We are currently investigating an issue impacting the Data Modeling service (**`datamodelstorage`**) in the** `az-arn-001` **cluster.**\n\n**A large spike in traffic against Data Modeling endpoints has led to an elevated rate of 503 errors (affecting roughly 3% of requests) in this cluster. Users and services interacting with Data Modeling endpoints may experience intermittent request failures.**\n\n**Our engineering team is actively investigating the cause of the errors and working on mitigation steps. Further updates will be provided as more information becomes available**",
"created_at": "2026-09-11T07:20:32Z",
"display_at": "2026-09-11T07:20:32Z",
"id": "01M27NDSVKBWQQFPHHD483XK13",
"incident_id": "01M27NDSVKCS0JGAMJHPHH6RCM",
"status": "investigating",
"updated_at": "2026-09-11T07:20:32Z"
}
],
"name": "Degraded Data Modeling Availability in az-arn-001",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-18T19:18:43Z",
"status": "resolved",
"updated_at": "2026-09-18T19:18:43Z"
},
{
"created_at": "2026-09-06T06:17:20Z",
"id": "01M1TNTFDEZYYCMPHHA79MZPF1",
"impact": "none",
"incident_updates": [
{
"body": "Services are fully recovered and operating normally. The elevated 5XX error rates affecting the **Timeseries** and **Sequences** APIs on cluster **az-pnq-gp-001** between approximately **07:18 and 07:45 CEST** on **September 6, 2026** were resolved following auto-healing of the FoundationDB operator pod instability. Monitoring confirms all deployments remain healthy and stable, with no data loss incurred.",
"created_at": "2026-09-06T14:54:43Z",
"display_at": "2026-09-06T14:54:43Z",
"id": "01M1VKDTTEWE00K5V8D5M6JGRA",
"incident_id": "01M1TNTFDEZYYCMPHHA79MZPF1",
"status": "resolved",
"updated_at": "2026-09-06T14:54:43Z"
},
{
"body": "**We experienced elevated 5XX error rates affecting the Timeseries and Sequences APIs on cluster** az-pnq-gp-001 **between approximately 07:18 and 07:45 CEST. Services have recovered and all deployments are healthy, with no indication of data loss. The engineering team is actively monitoring the service to ensure continued stability while completing the investigation into the underlying cause.**",
"created_at": "2026-09-06T06:17:20Z",
"display_at": "2026-09-06T05:18:00Z",
"id": "01M1TNTFDEDCH5F1K3PDZRE854",
"incident_id": "01M1TNTFDEZYYCMPHHA79MZPF1",
"status": "investigating",
"updated_at": "2026-09-06T06:21:46Z"
}
],
"name": "Elevated error rates on Timeseries and Sequences APIs",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-06T14:54:43Z",
"status": "resolved",
"updated_at": "2026-09-06T14:54:43Z"
},
{
"created_at": "2026-09-06T11:34:36Z",
"id": "01M1V7ZDPNGS835SN3RV3K8NMR",
"impact": "major",
"incident_updates": [
{
"body": "This incident has been resolved. The issue causing HTTP 500/503 errors when searching or aggregating Data Models in `westeurope-1` has been mitigated, and service performance has fully recovered. All systems are operating normally.",
"created_at": "2026-09-06T14:51:56Z",
"display_at": "2026-09-06T14:51:56Z",
"id": "01M1VK8R2XDGXFT3QY44M6QCGS",
"incident_id": "01M1V7ZDPNGS835SN3RV3K8NMR",
"status": "resolved",
"updated_at": "2026-09-06T14:51:56Z"
},
{
"body": "This incident has been resolved. The issue causing HTTP 500/503 errors when searching or aggregating Data Models in `westeurope-1` has been mitigated, and service performance has fully recovered. All systems are operating normally.",
"created_at": "2026-09-06T14:51:37Z",
"display_at": "2026-09-06T14:51:37Z",
"id": "01M1VK855WCV2NQKYW08V2S3R1",
"incident_id": "01M1V7ZDPNGS835SN3RV3K8NMR",
"status": "investigating",
"updated_at": "2026-09-06T14:51:37Z"
},
{
"body": "The engineering team is investigating and fixing issues with the Data Modelling search and aggregate APIs in the westeurope-1 cluster. Customer impact includes increased error rates on DM APIs and in the data exploration app in Fusion, and delays before new data is available in search results.",
"created_at": "2026-09-06T11:34:36Z",
"display_at": "2026-09-06T11:34:36Z",
"id": "01M1V7ZDPNXC3AHQ6M28B7650Z",
"incident_id": "01M1V7ZDPNGS835SN3RV3K8NMR",
"status": "investigating",
"updated_at": "2026-09-06T11:34:36Z"
}
],
"name": "Users experiencing HTTP 500/503 errors when searching or aggregating Data Models",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-06T14:51:56Z",
"status": "resolved",
"updated_at": "2026-09-06T14:51:56Z"
},
{
"created_at": "2026-09-04T08:17:35Z",
"id": "01M1NQX83ATXG612QGCQTAFA91",
"impact": "major",
"incident_updates": [
{
"body": "• **Problem:** The production `propertygraph-search` Elasticsearch cluster in `westeurope-1` went red for roughly 10 minutes.\n\n• **Root cause:** Propertygraph syncer activity and cluster topology created excessive master-coordination overhead. PVC mount failures were also observed on Elasticsearch master-data pods, but were not confirmed as the original cause.\n\n• **Resolution:** Engineering Team scaled down the propertygraph syncers to recover cluster health, then restored every `propertygraph-sync-v3` deployment to one replica in stages. They also reduced node/master-coordination overhead and reconfigured all 15 master-data nodes with 19 GB heaps; the cluster reached zero unassigned shards and was considered stable.",
"created_at": "2026-09-04T17:58:17Z",
"display_at": "2026-09-04T17:58:17Z",
"id": "01M1PS4H5E1VRAX8MR6KY0PBQA",
"incident_id": "01M1NQX83ATXG612QGCQTAFA91",
"status": "resolved",
"updated_at": "2026-09-04T17:58:17Z"
},
{
"body": "The engineering team is investigating and fixing issues with the Data Modelling search and aggregate APIs in the westeurope-1 cluster. Customer impact includes increased error rates on DM APIs and in the data exploration app in Fusion, and delays before new data is available in search results.",
"created_at": "2026-09-04T10:29:30Z",
"display_at": "2026-09-04T10:29:30Z",
"id": "01M1NZERWYRFAZD72QW7ZDSK3D",
"incident_id": "01M1NQX83ATXG612QGCQTAFA91",
"status": "monitoring",
"updated_at": "2026-09-04T10:29:30Z"
},
{
"body": "The engineering team is investigating and fixing issues with the Data Modelling search and aggregate APIs in the westeurope-1 cluster. Customer impact includes increased error rates on DM APIs and in the data exploration app in Fusion, and delays before new data is available in search results.",
"created_at": "2026-09-04T08:18:27Z",
"display_at": "2026-09-04T08:18:27Z",
"id": "01M1NQYTWZWDD09KVYMKXKQ6GB",
"incident_id": "01M1NQX83ATXG612QGCQTAFA91",
"status": "monitoring",
"updated_at": "2026-09-04T08:18:27Z"
},
{
"body": "The engineering team is investigating and fixing issues with the Data Modelling search and aggregate APIs in the westeurope-1 cluster. Customer impact includes increased error rates on DM APIs and in the data exploration app in Fusion, and delays before new data is available in search results.",
"created_at": "2026-09-04T08:17:35Z",
"display_at": "2026-09-04T08:17:35Z",
"id": "01M1NQX83A56G8ER30Z8MW5552",
"incident_id": "01M1NQX83ATXG612QGCQTAFA91",
"status": "monitoring",
"updated_at": "2026-09-04T08:17:35Z"
}
],
"name": "Users experiencing HTTP 500/503 errors when searching or aggregating Data Models",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-04T17:58:17Z",
"status": "resolved",
"updated_at": "2026-09-04T17:58:17Z"
},
{
"created_at": "2026-09-03T14:51:01Z",
"id": "01M1KW0XV8G3RTHHGQ8EERF7R1",
"impact": "major",
"incident_updates": [
{
"body": "The issue causing login failures across Cognite Data Fusion, InField, and Maintain has been resolved. The underlying deployment change was rolled back, and normal service has been fully restored across all clusters.",
"created_at": "2026-09-03T15:05:12Z",
"display_at": "2026-09-03T15:05:12Z",
"id": "01M1KWTWEV3TF88A1G5R2QWHZ8",
"incident_id": "01M1KW0XV8G3RTHHGQ8EERF7R1",
"status": "resolved",
"updated_at": "2026-09-03T15:05:12Z"
},
{
"body": "**Identified** - We are investigating an issue affecting user logins across Cognite Data Fusion (Fusion), Infield, and Maintain. Some users may experience login failures displaying the error: `\"An HTTP line is larger than 4096 bytes\"`.\n\nA recent identity service update has been identified as the likely cause, and our engineering team is actively deploying a rollback to restore full login functionality.\n\nWe will provide another update soon as the rollback is completed.",
"created_at": "2026-09-03T14:51:01Z",
"display_at": "2026-09-03T14:51:01Z",
"id": "01M1KW0XV8RNJ3H3VTJ9FQ79C7",
"incident_id": "01M1KW0XV8G3RTHHGQ8EERF7R1",
"status": "monitoring",
"updated_at": "2026-09-03T14:51:01Z"
}
],
"name": "Login Issues in Cognite Applications (Fusion, Infield, Maintain)",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-03T15:05:12Z",
"status": "resolved",
"updated_at": "2026-09-03T15:05:12Z"
},
{
"created_at": "2026-09-01T12:05:28Z",
"id": "01M1EDRAZ2T9GJTFQYCTW1DQVM",
"impact": "major",
"incident_updates": [
{
"body": "**Summary:** We experienced an issue affecting the Property Graph search service in the West Europe region. The Elasticsearch cluster became unstable after receiving unusually large synchronization requests, which caused circuit-breaker errors.\n\n**Steps taken:** We stabilized the Elasticsearch cluster, then re-enabled background synchronization in controlled stages while monitoring cluster health, CPU usage, and indexing rates.\n\n**Resolved status:** The service has stabilized and normal functionality has been restored. Background synchronization has resumed, and we will continue monitoring to ensure it remains healthy.",
"created_at": "2026-09-01T18:46:49Z",
"display_at": "2026-09-01T18:32:00Z",
"id": "01M1F4Q7DHDYS9FNBQ0BJNWW15",
"incident_id": "01M1EDRAZ2T9GJTFQYCTW1DQVM",
"status": "resolved",
"updated_at": "2026-09-01T18:47:23Z"
},
{
"body": "We have implemented mitigation steps for the issue impacting Data Model (DM) Search and aggregation queries in the westeurope-1 region. The service has stabilized, and we are currently monitoring the system to ensure full recovery.\n\nSearch and aggregation requests for data models in westeurope-1 should now complete successfully without HTTP 503 errors. Customers may still see slight delays in data freshness as background sync operations catch up, but normal functionality has been largely restored. Core data storage and other regions continue to operate normally.\n\nOur engineering team has successfully stabilized the underlying Elasticsearch cluster and carefully re-enabled background sync operations in controlled stages. We will continue to closely monitor cluster health and performance metrics to ensure the service remains stable.",
"created_at": "2026-09-01T18:40:09Z",
"display_at": "2026-09-01T18:30:00Z",
"id": "01M1F4B1N238EJ9EZFTQFPY7PQ",
"incident_id": "01M1EDRAZ2T9GJTFQYCTW1DQVM",
"status": "monitoring",
"updated_at": "2026-09-01T18:43:02Z"
},
{
"body": "**Summary:**\nWe are currently investigating an issue impacting Data Model (DM) Search and aggregation queries in the `westeurope-1` region.\n\n**Customer Impact:**\nRequests to search or aggregate data models in `westeurope-1` may fail with HTTP 503 errors or return incomplete/delayed results. Core data storage and other regions are operating normally.\n\n**Current Actions:**\nOur engineering team is actively working on cluster recovery and load mitigation, including throttling background sync operations to restore search performance.\n\n**Next Update:**\nWe are monitoring recovery progress and will provide another update within 60 minutes or as soon as significant updates are available.",
"created_at": "2026-09-01T12:05:28Z",
"display_at": "2026-09-01T12:05:28Z",
"id": "01M1EDRAZ2ZPFTKYSXRC59WR33",
"incident_id": "01M1EDRAZ2T9GJTFQYCTW1DQVM",
"status": "investigating",
"updated_at": "2026-09-01T12:05:28Z"
}
],
"name": "Degraded Data Model Search \u0026 Aggregations in westeurope-1",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-09-01T18:32:00Z",
"status": "resolved",
"updated_at": "2026-09-01T18:46:49Z"
},
{
"created_at": "2026-08-24T13:52:34Z",
"id": "01M0T0PPQ604AA0CXBAWPVYMGH",
"impact": "none",
"incident_updates": [
{
"body": "The issue got fixed on 27th August. All Apps are now up and running.",
"created_at": "2026-08-31T12:07:03Z",
"display_at": "2026-08-31T12:07:03Z",
"id": "01M1BVEGM9QMYSPFKMC285JF4H",
"incident_id": "01M0T0PPQ604AA0CXBAWPVYMGH",
"status": "resolved",
"updated_at": "2026-08-31T12:07:03Z"
},
{
"body": "We have successfully resolved the underlying deployment issue. Our engineering team corrected the production deployment configuration, ensuring it now points to the correct Firebase project and hosting site. The fix has been verified across multiple production clusters and confirmed to be working as expected.\nIf you had previously applied the temporary workaround of adding `ssl` to your application's **INSTALLED PACKAGES**, no further action is needed this entry can remain in place and will not interfere with the platform fix.",
"created_at": "2026-08-27T11:06:43Z",
"display_at": "2026-08-27T11:06:43Z",
"id": "01M11ED67TFVY205796CMRYJPR",
"incident_id": "01M0T0PPQ604AA0CXBAWPVYMGH",
"status": "monitoring",
"updated_at": "2026-08-27T11:06:43Z"
},
{
"body": "The issue leading to the `ModuleNotFoundError: No module named 'ssl'` error in Streamlit apps has been identified as a change in a third party library, specifically the Microsoft Authentication Library for Python (MSAL). MSAL is commonly used in conjuction with the Cognite SDK to handle authentication.\n\nMSAL version 1.38, released on 2026-08-24, references the `ssl` module without declaring a dependency on it. This works on ordinary CPython installations because that environment includes the `ssl` module by default.\n\nHowever, Streamlit in the browser uses the Pyodide runtime environment. Pyodide does not include the `ssl` module by default. This leads to failures when the latest MSAL library is used in Streamlit apps.\n\nThe engineering team is currently adding a workaround to the Cognite Streamlit runtime environment setup to ensure that the `ssl` module is always present.",
"created_at": "2026-08-25T07:44:39Z",
"display_at": "2026-08-25T07:44:39Z",
"id": "01M0VY1QQ14FKZ97K1XAF9SCEX",
"incident_id": "01M0T0PPQ604AA0CXBAWPVYMGH",
"status": "identified",
"updated_at": "2026-08-25T07:44:39Z"
},
{
"body": "We are investigating issues with CDF-hosted Streamlit apps, where apps get the error **ModuleNotFoundError: No module named 'ssl'**\nWe are investigating the issue. The issue can affect all users of Streamlit apps regardless of which CDF clusters they are hosted in. In the meantime, explicitly adding the \"ssl\" module to your app's Python requirements can fix the immediate problem and is worth trying as a workaround.\n\nNote: CDF-hosted Streamlit apps are in [public preview](https://docs.cognite.com/cdf/streamlit/index#streamlit-apps) and are not yet in [production](https://docs.cognite.com/cdf/product_feature_status#public-preview).",
"created_at": "2026-08-24T13:52:34Z",
"display_at": "2026-08-24T13:52:34Z",
"id": "01M0T0PPQ6JD26AEEQXPHRSVGC",
"incident_id": "01M0T0PPQ604AA0CXBAWPVYMGH",
"status": "investigating",
"updated_at": "2026-08-24T13:52:34Z"
}
],
"name": "CDF hosted Streamlit apps failing due to missing \"ssl\" dependency",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-08-31T12:07:03Z",
"status": "resolved",
"updated_at": "2026-08-31T12:07:03Z"
},
{
"created_at": "2026-08-19T17:37:18Z",
"id": "01M0DHJKAG6DC2N9BCFVGV4V30",
"impact": "critical",
"incident_updates": [
{
"body": "Issue is fixed, and no further degradations were observed.",
"created_at": "2026-08-20T09:06:51Z",
"display_at": "2026-08-20T09:06:51Z",
"id": "01M0F6RNR7324HY6WKPYS6C8DJ",
"incident_id": "01M0DHJKAG6DC2N9BCFVGV4V30",
"status": "resolved",
"updated_at": "2026-08-20T09:06:51Z"
},
{
"body": "The database issue has been resolved and services are recovering. We are monitoring the situation.",
"created_at": "2026-08-19T18:02:45Z",
"display_at": "2026-08-19T17:46:00Z",
"id": "01M0DK16D70XDJ96GFGG5KR5A5",
"incident_id": "01M0DHJKAG6DC2N9BCFVGV4V30",
"status": "monitoring",
"updated_at": "2026-08-19T18:03:14Z"
},
{
"body": "We are aware of a service outage affecting the CDF cluster westeurope-1. The issue has severe impact on some but not all of the projects in cluster and are caused by a database server outage after a routine maintenance operation. Cognite's engineering team is actively working with our cloud provider to resolve the issue.",
"created_at": "2026-08-19T17:37:18Z",
"display_at": "2026-08-19T16:52:00Z",
"id": "01M0DHJKAGTNYZSG3N8XR17NR8",
"incident_id": "01M0DHJKAG6DC2N9BCFVGV4V30",
"status": "investigating",
"updated_at": "2026-08-19T17:39:46Z"
}
],
"name": "Partial service outage affecting CDF westeurope-1",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-08-20T09:06:51Z",
"status": "resolved",
"updated_at": "2026-08-20T09:06:51Z"
},
{
"created_at": "2026-08-13T19:33:28Z",
"id": "01KZY9V0M8HV20XE8FTXJR0AZY",
"impact": "minor",
"incident_updates": [
{
"body": "This incident has been resolved. It is operating normally. We apologize for any inconvenience and thank you for your patience.",
"created_at": "2026-08-14T12:44:39Z",
"display_at": "2026-08-14T12:44:39Z",
"id": "01M004V595J3YJK7ZPFFNTNQ2R",
"incident_id": "01KZY9V0M8HV20XE8FTXJR0AZY",
"status": "resolved",
"updated_at": "2026-08-14T12:44:39Z"
},
{
"body": "Functions requests on az-dsm-psaas-001 returned 500s for a temporary period between 19:25 and 19:40 CEST. The immediate impact was resolved at 19:40 CET / 17:40 UTC, and we are now monitoring the clusters to ensure continued stability.",
"created_at": "2026-08-13T19:34:50Z",
"display_at": "2026-08-13T17:40:00Z",
"id": "01KZY9XGG5YFT1JQZ1N3E59PY6",
"incident_id": "01KZY9V0M8HV20XE8FTXJR0AZY",
"status": "monitoring",
"updated_at": "2026-08-13T19:37:22Z"
},
{
"body": "We are currently investigating an elevated rate of 5xx errors affecting the Functions API. The issue appears to be caused by an upstream dependency disruption (Azure Key Vault RBAC loading timeout) in the Central US region, rather than a direct cluster-level outage. The connection rates are already stabilizing, and we are monitoring the recovery.",
"created_at": "2026-08-13T19:33:28Z",
"display_at": "2026-08-13T17:25:00Z",
"id": "01KZY9V0M89YZHQ60BXXCMQTS3",
"incident_id": "01KZY9V0M8HV20XE8FTXJR0AZY",
"status": "investigating",
"updated_at": "2026-08-13T19:37:02Z"
}
],
"name": "Functions API - Degraded Performance",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-08-14T12:44:39Z",
"status": "resolved",
"updated_at": "2026-08-14T12:44:39Z"
},
{
"created_at": "2026-08-11T10:14:35Z",
"id": "01KZR527988ZK33KSH0PYMXSEF",
"impact": "critical",
"incident_updates": [
{
"body": "This incident has been resolved. Cogshop service is up and operating normally. We apologize for any inconvenience and thank you for your patience.",
"created_at": "2026-08-11T13:41:28Z",
"display_at": "2026-08-11T13:41:28Z",
"id": "01KZRGX0RR5TSR6Z82WA1JXHWV",
"incident_id": "01KZR527988ZK33KSH0PYMXSEF",
"status": "resolved",
"updated_at": "2026-08-11T13:41:28Z"
},
{
"body": "We identified the issue as a missing TLS certificate in the `wildcard-cognite-ai-tls-patch` secret. Access to the `powerops-certmgr-vault` key vault was updated to allow regeneration/import of the certificate, and we can now see the certificate has been recreated.\n\nThe Cogshop endpoint is responding again: [shop-api.az-inso-powerops.cognite.ai/docs](https://shop-api.az-inso-powerops.cognite.ai/docs). We are continuing to monitor the service closely to confirm it remains stable before marking this incident as resolved.",
"created_at": "2026-08-11T13:19:15Z",
"display_at": "2026-08-11T13:19:15Z",
"id": "01KZRFMBPP81N7DC2AZ8BZT9B7",
"incident_id": "01KZR527988ZK33KSH0PYMXSEF",
"status": "monitoring",
"updated_at": "2026-08-11T13:19:15Z"
},
{
"body": "We are currently investigating an issue causing the Cogshop service to be unavailable. Our engineering team has identified the root cause and is actively working on a fix. We will provide further updates as soon as more information is available.",
"created_at": "2026-08-11T10:14:35Z",
"display_at": "2026-08-11T10:14:35Z",
"id": "01KZR527988XWDSJ98YHXH4WGA",
"incident_id": "01KZR527988ZK33KSH0PYMXSEF",
"status": "investigating",
"updated_at": "2026-08-11T10:14:35Z"
}
],
"name": "Cogshop Service Disruption",
"page_id": "01H16FDGE21SXKCBJCZJVXEHSX",
"resolved_at": "2026-08-11T13:41:28Z",
"status": "resolved",
"updated_at": "2026-08-11T13:41:28Z"
}
],
"page": {
"id": "01H16FDGE21SXKCBJCZJVXEHSX",
"name": "Cognite Service",
"updated_at": "2026-11-13T12:26:30Z",
"url": "https://status.cognite.com/"
}
}