Snapshot 311553
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"nxbqtxqrm55f",
"xs5vhwly9mk5",
"f347vm5yn95f",
"vxl2rz20dcyr",
"yyv8hjtmvpzv",
"cc654c3j6476",
"kzjygtpl015t",
"n0m4jk53hrym",
"5znhlv1nwfcf",
"83n5xv3hpb69",
"by80qyspb2jq",
"sn84zlkwcg24",
"lv49p6nbl193",
"5qd852klgxc0",
"15bv7h1knvsk",
"mn3934kz3w69",
"5rys3t6403zn",
"1lccfpfwmh4y",
"rz6k0bcnrpr1",
"vy68z1vstc62",
"l539c2tfydzj",
"8vf2zpmhfyy0",
"j4tq5500kmg8",
"77bh7sgmbbv3"
],
"created_at": "2021-06-01T14:46:03.030-07:00",
"description": null,
"group_id": null,
"id": "27qq03xf2msg",
"name": "Prod 1",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.090-07:00"
},
{
"components": [
"s54kkcr6t0gq",
"gbltvmpdkb3r",
"bqhhx747whp3",
"tgcylzz5nlry",
"c6myzvwng2lw",
"9677ldz5y7sd",
"hp77w1mwh2xg",
"ctmk1q1dw7nr",
"qvs27pfch8vy",
"65kdr9mrlblc",
"cg2xjl1jzk0m",
"qbzy2sc4skqh",
"gdlfxhr31qmb",
"sbwzykrf1q8t",
"b4tvy5s9wr6z",
"v1slhlqmmbm1",
"5k3yywp3xvqj",
"640z11cmjv3n",
"fwh6z4glx958",
"0yrdl70cplw9",
"czfjkrnqlcsk",
"9pr1l3rwf14n",
"8tnc91hlt8d2"
],
"created_at": "2021-06-01T14:53:31.712-07:00",
"description": null,
"group_id": null,
"id": "qbxn0mh45jhv",
"name": "Prod 2",
"page_id": "zmnt6tkys0q0",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.103-07:00"
},
{
"components": [
"rwx7y5zv4yq5",
"l1sx8bck1yzb",
"fw0zmk94bkpc",
"tq8rdbq9np4d",
"brmlw2942fbc",
"4fpjqbv7k6fp",
"pb161tg7ccrs",
"th62w3qy4r3m",
"kxxh18chwqlg",
"k4sqhp5t769d",
"nw83hg1v0ftp",
"q489v85zw6qs",
"vf6qxw4l52m4",
"n0f6mg62hgx3",
"ths5wq122vtx",
"804lzkqf2vfk",
"jjmrjld2vwnb",
"yhy2w4n4q7kk",
"sbm4gmjj60nl",
"g5xm7jlw5rr4",
"ztrdyp8d48sx",
"mbpd7mcgc0l0"
],
"created_at": "2021-06-01T14:54:00.058-07:00",
"description": null,
"group_id": null,
"id": "z7txn2324j6k",
"name": "Prod 3",
"page_id": "zmnt6tkys0q0",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.115-07:00"
},
{
"components": [
"p9t9g01vgy47",
"5161jzbx0d8w",
"j9qkdrxdylw2",
"bp1hsfllmds0",
"czhk96jtx93b",
"rgdxklp8p0kd",
"trkh53c2bscy",
"sfc57k1p0sqp",
"790nrgxflg5x",
"k0hbqz5c81v8",
"h1wfypqglf49",
"gvhsp1j79nbm",
"xzg7lrrqyk37",
"nxsthnf9gk3g",
"6xhczbg7xfyv",
"0vlv5pjpjzbs",
"561g9b7sqgct",
"jqz7n41m65f7"
],
"created_at": "2024-07-10T21:08:30.585-07:00",
"description": null,
"group_id": null,
"id": "x1089mr22815",
"name": "Prod 4",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.126-07:00"
},
{
"components": [
"zbyg84df96x7",
"007h1ztzxtlw",
"v7mz6ltk413q",
"w3csxhc0mqz7",
"rbn7hjb2qf8r",
"y2wg26xx8r7r",
"0kkbcr1h3cvx",
"x3wy7hhj591c",
"z9267hzwpnwq",
"fwvncn7t50n9",
"r87dg7ccrjpv",
"lg2vtjgxqkzw",
"y3rkhygkv1g3",
"1b70wzgp8spj",
"3hdhfjy41l7p",
"rh8y3gywr698",
"zp7rk1rtt25b",
"yhy0vm97y09n",
"mf7jrz36jy6b"
],
"created_at": "2024-09-09T21:49:39.223-07:00",
"description": null,
"group_id": null,
"id": "6y3np02bqtxf",
"name": "Prod Eu 1",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-22T04:49:45.778-07:00"
},
{
"created_at": "2021-12-10T16:29:47.633-08:00",
"description": "Harness OverOps",
"group_id": null,
"id": "1h5qbtxd2622",
"name": "Service Reliability Management - Error Tracking FirstGen (fka OverOps)",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-12-10T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2024-10-28T19:09:00.147-07:00"
},
{
"components": [
"52zt8q9l5yk4",
"wjt25yrgfl1d"
],
"created_at": "2025-01-13T08:34:11.290-08:00",
"description": null,
"group_id": null,
"id": "rx8vt1qyrpsv",
"name": "Software Engineering Insights FirstGen (fka Propelo)",
"page_id": "zmnt6tkys0q0",
"position": 10,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-01-13T08:34:11.290-08:00"
},
{
"components": [
"hnbf7sq0cq5b",
"s0dh2g03zz7r",
"fbbt3z4ytn23",
"yxvnmfxvggb9",
"0r0gmbxty2d9",
"dnkfpsn0r5vr"
],
"created_at": "2025-09-01T23:44:00.402-07:00",
"description": "Traceable Clusters",
"group_id": null,
"id": "np782snlmj00",
"name": "Traceable",
"page_id": "zmnt6tkys0q0",
"position": 11,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-09-01T23:44:29.556-07:00"
},
{
"components": [
"qzlxgjbrdby4",
"gflflybxx6b8",
"zwf5f0yg33qk",
"w90818668zwc",
"xwbxwtcl9g25",
"y21hwc1hhpnz",
"qcj53m9f1ptq",
"6lmbj16bvn16"
],
"created_at": "2025-11-05T10:12:41.367-08:00",
"description": "Feature Management \u0026 Experimentation",
"group_id": null,
"id": "wm26z10lksx3",
"name": "Feature Management \u0026 Experimentation (FME)",
"page_id": "zmnt6tkys0q0",
"position": 13,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-02-04T14:36:46.108-08:00"
},
{
"components": [
"s1dlyyndwjl6",
"6k06w76wqwsc",
"p20hnvtnpw1b",
"drq1sbn8hw11",
"v5rf21nbqwg2",
"nv3wkg4bf6kp"
],
"created_at": "2026-01-06T02:08:28.178-08:00",
"description": "Dedicated SAAS cluster for Goldman Sachs",
"group_id": null,
"id": "2w5qsvz2lk4n",
"name": "Prod 7",
"page_id": "zmnt6tkys0q0",
"position": 14,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-01-06T02:11:59.702-08:00"
},
{
"components": [
"0rxrdlk4dr3m",
"snxfjcdrnddd",
"lfrwy26bjy27",
"s9f6pwt8tbj1",
"gl4wwd0s6l4n",
"7jfnyljgkhsc",
"fgy1h37xbbjg"
],
"created_at": "2026-01-06T02:35:45.999-08:00",
"description": "Dedicated SAAS cluster for Barclays",
"group_id": null,
"id": "n6rk38cphg56",
"name": "Prod 8",
"page_id": "zmnt6tkys0q0",
"position": 15,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-01-06T02:36:31.092-08:00"
},
{
"components": [
"shg48w7t35l8",
"1zmt4tlpxh7d",
"tsvcht7ggxpq",
"6q2y478qnshb",
"5vjk880lv3yb",
"vqy2pfsw7sxx",
"gfsntx0t1n45",
"glxwgv0pdjrb",
"6cq3s06svg8n",
"d7b0scq7bvf5",
"xckv4nnb4hjy",
"t9pvbwy0smfp",
"m25krg585c3z",
"7fp2gfxytcw0",
"l5np7vy2h0bc",
"kbjf89kfnxfm",
"z1r28087zxf5",
"gm8yq9pr4xbk",
"47jx56p992v7",
"j4mnhf18ky22",
"93j99d3r0nb3",
"mttlsgnqy0mw",
"1yjpjqbtb1xn"
],
"created_at": "2026-04-14T01:59:10.412-07:00",
"description": null,
"group_id": null,
"id": "j9f15dxpqydw",
"name": "Dev",
"page_id": "zmnt6tkys0q0",
"position": 16,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-11T07:30:50.641-07:00"
},
{
"components": [
"tgbvl8jm69gd",
"n48l3qtwftyq",
"9nh0xfkz9hkq",
"jddlj6k9xs29",
"pwv8nvc5d0cz"
],
"created_at": "2026-04-20T02:54:03.635-07:00",
"description": null,
"group_id": null,
"id": "23r8514jn030",
"name": "Prod 5",
"page_id": "zmnt6tkys0q0",
"position": 17,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-04-20T02:55:08.885-07:00"
},
{
"created_at": "2026-06-19T02:52:03.505-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Dev",
"group_id": null,
"id": "9r4nnf9jzl4t",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-06-19T02:52:03.505-07:00"
},
{
"components": [
"y41n6v53x5z1",
"hbt1g5bqvg1l",
"hjr9dqb9t38t",
"th0ygdtzlvdk",
"t45ncxygrpxm",
"nyhhbjtgtst4",
"8mjkl07z9kn1",
"t8gybmf0b2qt",
"q1369hjq7nn4",
"60cy4p5vr6gp",
"xbqdy20jrrs5",
"skccvt241051",
"52ny1ymph3wd",
"2cn1374658lx",
"r09j2jvg6r3h",
"l0fd5d66jnpp",
"kryxb6g45sqn",
"lbdf8v2kkq2j",
"hg36cdlm95bl",
"fhb7jrwcfdzl",
"w9kcl4xznsym",
"jdc2stt9jvds",
"kf7xlcfh77bx",
"4blb94xkwpf3"
],
"created_at": "2026-07-22T22:21:45.295-07:00",
"description": null,
"group_id": null,
"id": "1fpldmj5f61k",
"name": "Prod 6",
"page_id": "zmnt6tkys0q0",
"position": 19,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-22T22:21:45.295-07:00"
}
],
"incidents": [
{
"components": [
{
"created_at": "2022-03-15T18:10:49.160-07:00",
"description": "Harness Continuous Delivery Next Generation (CDNG) in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "gbltvmpdkb3r",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": true,
"start_date": "2021-10-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-11T22:07:00.632-08:00"
},
{
"created_at": "2023-08-21T10:47:38.568-07:00",
"description": "Harness Continuous Integration Enterprise (CIE) - Self Hosted Runners in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "9677ldz5y7sd",
"name": "Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2023-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-07-12T23:41:30.165-07:00"
},
{
"created_at": "2021-06-01T14:56:05.591-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "hp77w1mwh2xg",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-04-10T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.931-07:00"
},
{
"created_at": "2024-11-07T20:33:35.371-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "ctmk1q1dw7nr",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.484-07:00"
},
{
"created_at": "2024-11-07T20:39:41.158-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod2",
"group_id": "qbxn0mh45jhv",
"id": "qvs27pfch8vy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.900-07:00"
},
{
"created_at": "2022-04-08T09:16:22.823-07:00",
"description": "Security Testing Orchestration (STO) in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "qbzy2sc4skqh",
"name": "Security Testing Orchestration (STO)",
"page_id": "zmnt6tkys0q0",
"position": 12,
"showcase": true,
"start_date": "2022-03-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-07-12T23:42:01.202-07:00"
},
{
"created_at": "2024-04-01T19:28:09.598-07:00",
"description": "Harness Internal Developer Portal (IDP) in Prod2",
"group_id": "qbxn0mh45jhv",
"id": "sbwzykrf1q8t",
"name": "Internal Developer Portal (IDP)",
"page_id": "zmnt6tkys0q0",
"position": 14,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-07-12T23:42:12.056-07:00"
},
{
"created_at": "2024-04-17T02:38:45.540-07:00",
"description": "Harness Infrastructure as Code Management (IaCM) in Prod2",
"group_id": "qbxn0mh45jhv",
"id": "b4tvy5s9wr6z",
"name": "Infrastructure as Code Management (IaCM)",
"page_id": "zmnt6tkys0q0",
"position": 15,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-08-26T02:05:46.010-07:00"
},
{
"created_at": "2024-07-31T21:13:51.477-07:00",
"description": "Harness Software Supply Chain Assurance (SSCA) in Prod2",
"group_id": "qbxn0mh45jhv",
"id": "v1slhlqmmbm1",
"name": "Software Supply Chain Assurance (SSCA)",
"page_id": "zmnt6tkys0q0",
"position": 16,
"showcase": true,
"start_date": "2024-07-31T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-07-12T23:42:23.988-07:00"
}
],
"created_at": "2026-11-12T21:12:58.854-08:00",
"id": "hssx3qktqbmt",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "gbltvmpdkb3r",
"name": "Prod 2 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "9677ldz5y7sd",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qbzy2sc4skqh",
"name": "Prod 2 - Security Testing Orchestration (STO)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "sbwzykrf1q8t",
"name": "Prod 2 - Internal Developer Portal (IDP)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "b4tvy5s9wr6z",
"name": "Prod 2 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "v1slhlqmmbm1",
"name": "Prod 2 - Software Supply Chain Assurance (SSCA)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We will be undergoing cluster maintenance in Prod2 for improved stability and scalability. There would be an impact to inflight pipelines during this maintenance.",
"created_at": "2026-11-12T21:12:58.990-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-12T21:12:58.990-08:00",
"id": "863rsyf5my7l",
"incident_id": "hssx3qktqbmt",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-12T21:12:58.990-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Upcoming maintenance for Prod2",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": null,
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-12-12T20:00:00.000-08:00",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-12-12T22:00:00.000-08:00",
"shortlink": "https://stspg.io/g09wzdfgb5wt",
"started_at": "2026-11-12T21:12:58.823-08:00",
"status": "scheduled",
"updated_at": "2026-11-12T21:12:58.994-08:00"
},
{
"components": [
{
"created_at": "2025-08-27T10:30:51.938-07:00",
"description": "Core platform constructs eg. Platform delegate (eg. Delegate communication), Platform Core (eg. Account, Login, Project/Org, Connectors, Secrets, Delegate, Settings, Notifications, Audits, Templates, Services, Environments , Policies, File Store, Log Uploads), Platform Authentication \u0026 Authorization (login, RBAC, logout, API keys, Service account, Tokens etc.)",
"group_id": "qbxn0mh45jhv",
"id": "0yrdl70cplw9",
"name": "Platform",
"page_id": "zmnt6tkys0q0",
"position": 20,
"showcase": true,
"start_date": "2025-08-27T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-19T22:48:14.129-08:00"
}
],
"created_at": "2026-11-19T13:41:06.822-08:00",
"id": "t3j7zc15fr05",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "0yrdl70cplw9",
"name": "Prod 2 - Platform",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-11-19T22:48:14.169-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-19T22:48:14.169-08:00",
"id": "8n0k7j4zk3zs",
"incident_id": "t3j7zc15fr05",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T22:48:14.169-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "0yrdl70cplw9",
"name": "Prod 2 - Platform",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are continuing to monitor for any further issues.",
"created_at": "2026-11-19T13:54:26.324-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-19T13:54:26.324-08:00",
"id": "k3v1wqrgqd4p",
"incident_id": "t3j7zc15fr05",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T13:54:26.324-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "0yrdl70cplw9",
"name": "Prod 2 - Platform",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-11-19T13:53:38.527-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-19T13:53:38.527-08:00",
"id": "1tvr8s760r0d",
"incident_id": "t3j7zc15fr05",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T13:53:38.527-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "0yrdl70cplw9",
"name": "Prod 2 - Platform",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Customers have reported problems accessing Prod 2 accounts. Customers may see that they are stuck attempting to load.",
"created_at": "2026-11-19T13:41:06.894-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-19T13:41:06.894-08:00",
"id": "333km0cnvvt9",
"incident_id": "t3j7zc15fr05",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-19T13:41:06.894-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-19T13:53:38.527-08:00",
"name": "Customers unable to log into Prod 2",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-19T22:48:14.169-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/1x0hqhv4rtc8",
"started_at": "2026-11-19T13:41:06.811-08:00",
"status": "resolved",
"updated_at": "2026-11-19T22:48:14.190-08:00"
},
{
"components": [
{
"created_at": "2022-03-15T17:37:45.978-07:00",
"description": "Harness Continuous Delivery Next Generation (CDNG) in Prod 3",
"group_id": "z7txn2324j6k",
"id": "l1sx8bck1yzb",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": true,
"start_date": "2021-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-11-13T10:42:18.591-08:00"
}
],
"created_at": "2026-11-13T09:56:09.939-08:00",
"id": "k5nnlk8ynq11",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-11-13T10:42:18.629-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-13T10:42:18.629-08:00",
"id": "25z241vkfrw0",
"incident_id": "k5nnlk8ynq11",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-13T10:42:18.629-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-11-13T10:12:40.653-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-13T10:12:40.653-08:00",
"id": "w4rcnblpqgyh",
"incident_id": "k5nnlk8ynq11",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-13T10:12:40.653-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented, We are checking for any failed pipelines in parallel.",
"created_at": "2026-11-13T10:00:17.046-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-13T10:00:17.046-08:00",
"id": "9pp34ntv6fdp",
"incident_id": "k5nnlk8ynq11",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-13T10:00:17.046-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating a slowness on prod3 environment, will continue to post further updates.",
"created_at": "2026-11-13T09:56:10.037-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-13T09:56:10.037-08:00",
"id": "vlb5wn426jw7",
"incident_id": "k5nnlk8ynq11",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-13T09:56:10.037-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-13T10:12:40.653-08:00",
"name": "Pipeline slowness on prod3",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "# 1. Summary\n\nOn November 13, 2026, Prod3 pipelines experienced slowness, failed to start, or hung f, with some executions automatically aborted after being detected as stuck for more than 5 minutes. The issue was triggered by a single customer pipeline with an unusually large number of concurrent stages , which generated many simultaneous updates to a single internal data structure used to track that pipeline's execution state. This saturated the CPU and write capacity of the database node backing pipeline orchestration in Prod3, slowing down pipeline processing platform-wide. The underlying condition resolved on its own once the triggering pipeline's updates completed; a permanent architectural fix that spreads this execution data across multiple documents is in progress.\n\n# 2. Impact\n\n* Pipelines in Prod3 were slow, failed to start, or hung .\n* Some executions were automatically aborted after being detected as hung for more than 5 minutes.\n* Two customers reported being affected by the slowdown.\n\n# 3. Root Cause\n\nOur pipeline platform tracks the state of a pipeline execution's steps in a single internal data structure \\(document\\) per execution. Every step in an execution writes its progress into that same document.\n\nOne customer's pipeline ran with an unusually large number of concurrent stages Because all of these stages were updating the same underlying document for their shared execution, the resulting writes began to conflict with one another. Each conflicting write had to be retried, and as the queue of pending writes grew, operations started timing out an internal lock-wait window \\( and being resubmitted to the database directly. \n\nAs the triggering pipeline grew in size, its state document also grew larger, which made each individual update slower and further compounded the contention.\n\n# 4. Mitigation\n\n* The underlying condition resolved on its own once the triggering pipeline's long-running updates finished, at which point CPU usage and write latency returned to normal and the backlog of delayed pipeline events cleared.\n* No manual scale-up, restart, or failover was required to restore service as recovery was already underway by the time responders isolated the triggering pipeline as the cause.\n* As an interim safeguard, we increased the internal lock-wait window , which reduces \\(though does not eliminate\\) the chance of a large number of concurrent writers piling up on the same execution document.\n\n# 5. Next Steps\n\n1. **Roll out an architectural change** that splits a pipeline execution's state across multiple documents, sharded by stage, instead of storing it in a single document per execution — so no single execution can concentrate this much write contention on one document or database primary, regardless of its size.\n2. **Enhance monitoring and alerting** that can identify an individual execution driving disproportionate database write load, so this class of issue can be detected and addressed before it affects other customers' pipelines.\n3. **Review and tune lock-wait and retry behavior** for this execution-state update path more broadly, beyond the interim timeout increase already applied.\n4. **Evaluate guardrails for pipelines with very high stage concurrency** \n\n_We appreciate your patience while we worked through a complete resolution, and we remain committed to strengthening the resilience of our pipeline orchestration infrastructure as a result of this incident._",
"postmortem_body_last_updated_at": "2026-11-17T23:50:03.753-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-17T23:50:03.753-08:00",
"reminder_intervals": null,
"resolved_at": "2026-11-13T10:42:18.629-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/ywg57gflpvq3",
"started_at": "2026-11-13T09:56:09.931-08:00",
"status": "resolved",
"updated_at": "2026-11-17T23:50:03.782-08:00"
},
{
"components": [
{
"created_at": "2024-07-10T21:08:30.607-07:00",
"description": "Global Gateway intermittently encountering connection errors in Prod 4",
"group_id": "x1089mr22815",
"id": "p9t9g01vgy47",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 1,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-11T22:07:00.552-08:00"
},
{
"created_at": "2024-09-09T21:49:39.244-07:00",
"description": "Harness Continuous Delivery Next Generation (CDNG) in Prod Eu1",
"group_id": "6y3np02bqtxf",
"id": "zbyg84df96x7",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 1,
"showcase": true,
"start_date": "2024-09-09T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-11T22:07:00.581-08:00"
},
{
"created_at": "2022-03-15T17:37:45.978-07:00",
"description": "Harness Continuous Delivery Next Generation (CDNG) in Prod 3",
"group_id": "z7txn2324j6k",
"id": "l1sx8bck1yzb",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": true,
"start_date": "2021-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-11-13T10:42:18.591-08:00"
},
{
"created_at": "2022-03-15T18:10:49.160-07:00",
"description": "Harness Continuous Delivery Next Generation (CDNG) in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "gbltvmpdkb3r",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": true,
"start_date": "2021-10-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-11T22:07:00.632-08:00"
},
{
"created_at": "2022-03-15T18:11:41.146-07:00",
"description": "Harness Continuous Delivery Next Generation (CDNG) in Prod 1",
"group_id": "27qq03xf2msg",
"id": "xs5vhwly9mk5",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": true,
"start_date": "2021-10-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-11T22:07:00.656-08:00"
}
],
"created_at": "2026-11-11T16:41:06.052-08:00",
"id": "h0mqzxn24wwj",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "xs5vhwly9mk5",
"name": "Prod 1 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "gbltvmpdkb3r",
"name": "Prod 2 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "zbyg84df96x7",
"name": "Prod Eu 1 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-11-11T22:07:00.696-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-11T22:07:00.696-08:00",
"id": "wljwggbfsv3v",
"incident_id": "h0mqzxn24wwj",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-11T22:07:00.696-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xs5vhwly9mk5",
"name": "Prod 1 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "gbltvmpdkb3r",
"name": "Prod 2 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "zbyg84df96x7",
"name": "Prod Eu 1 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-11-11T17:05:59.941-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-11T17:05:59.941-08:00",
"id": "v7n0tnxbg2sg",
"incident_id": "h0mqzxn24wwj",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-11T17:05:59.941-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xs5vhwly9mk5",
"name": "Prod 1 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "gbltvmpdkb3r",
"name": "Prod 2 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "zbyg84df96x7",
"name": "Prod Eu 1 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are still investigating the issue, Deployments that rely on the missing tools can experience failures.\n \nAs of now this only impacts the customers who have the delegates running on these versions.\n\n26.11.90407\n26.10.90316\n\n26.10.90313 and earlier versions are not impacted.",
"created_at": "2026-11-11T16:54:14.555-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-11T16:54:14.555-08:00",
"id": "zfytt73j7wm8",
"incident_id": "h0mqzxn24wwj",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-11T16:54:14.555-08:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "xs5vhwly9mk5",
"name": "Prod 1 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "gbltvmpdkb3r",
"name": "Prod 2 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "zbyg84df96x7",
"name": "Prod Eu 1 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-11-11T16:41:06.251-08:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-11-11T16:41:06.251-08:00",
"id": "b6z1vtn9kn3x",
"incident_id": "h0mqzxn24wwj",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-11T16:41:06.251-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-11T17:05:59.941-08:00",
"name": "Delegate version 26.11.90407/26.10.90316 missing client tools",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "## **Summary**\n\nOn Nov 12, 2027, Delegate team released a new version `26.11.90407` as well as critical fixes for several older versions where maximal delegate image didn’t include the standard tools that are normally packaged. This impacted customers that relied on these tools \\(like kubectl, helm etc…\\) in maximal image and that were using Delegate auto-upgrade \\(as Delegate automatically upgraded to the new version\\). We mitigated the issue by rolling back the auto-upgrade which brought delegates back to a stable version. The issue was fixed and new versions released as a final fix.\n\n## **Root Cause**\n\nDelegate build pipeline had a latent bug which was leaving the build environment in an inconsistent state after the build. Normally this didn’t have any effect on builds because the team was letting build failures fail the pipeline, so each build was starting from a fresh environment. Recently Delegate team started using Retry Failure Strategy for some steps to reduce impact of transient failures which inadvertently caused retry attempts to build from the tainted state. This resulted in the minimal Dockerfile being used for both image types during the build.\n\n## **Impact**\n\nEach cluster got new version in slightly different time. The main impact was on CD. Deployments were affected \\(e.g. could start failing due to lack of client tools or get assigned to different Delegates that weren’t upgraded\\) for customers that are on Delegate auto-upgrade.\n\n## **Remediation**\n\n* **Immediate:** Rolled back to the last known-good release, restoring pipeline functionality for customers.\n* **Permanent:** Fixed build pipeline issue and built a new versions of Delegates with all the required tools_._\n\n## **Action Items**\n\nTo prevent such issues from happening again, Harness will\n\n1. Optimize the Upgrader check interval from 1h down to 20min to help expedite version change especially in case of downgrades.\n2. Enhance the test suite by adding tool presence and version checks for each image type",
"postmortem_body_last_updated_at": "2026-11-19T22:54:45.631-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-19T22:54:45.631-08:00",
"reminder_intervals": null,
"resolved_at": "2026-11-11T22:07:00.696-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/085pdx9kv6lz",
"started_at": "2026-11-11T16:41:05.951-08:00",
"status": "resolved",
"updated_at": "2026-11-19T22:54:45.655-08:00"
},
{
"components": [],
"created_at": "2026-11-05T16:59:06.522-08:00",
"id": "14wd7nkgxlh4",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2026-11-05T23:13:30.657-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-05T23:13:30.657-08:00",
"id": "8qxphg54gjxp",
"incident_id": "14wd7nkgxlh4",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-05T23:13:30.657-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-11-05T17:16:16.936-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-05T17:16:16.936-08:00",
"id": "779zsv4s2ks9",
"incident_id": "14wd7nkgxlh4",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-05T17:16:16.936-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We are currently investigating this issue.",
"created_at": "2026-11-05T16:59:06.573-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-05T16:59:06.573-08:00",
"id": "3ws1c8ssdn5b",
"incident_id": "14wd7nkgxlh4",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-05T16:59:06.573-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "apidocs.harness.io not responding to HTTPS requests",
"page_id": "zmnt6tkys0q0",
"postmortem_body": ".",
"postmortem_body_last_updated_at": "2026-11-16T11:26:27.025-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-16T11:26:27.025-08:00",
"reminder_intervals": null,
"resolved_at": "2026-11-05T23:13:30.657-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/kftdhhdfp3pq",
"started_at": "2026-11-05T16:59:06.516-08:00",
"status": "resolved",
"updated_at": "2026-11-16T11:26:27.049-08:00"
},
{
"components": [],
"created_at": "2026-11-02T17:01:23.020-08:00",
"id": "96kl099bnr4f",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "The provider has resolved the issue and service is restored.",
"created_at": "2026-11-03T18:33:11.735-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T18:33:11.000-08:00",
"id": "p1nz748w0q59",
"incident_id": "96kl099bnr4f",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T18:33:51.618-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-11-03T04:41:37.652-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T04:41:37.652-08:00",
"id": "vbxgbm6p8rjq",
"incident_id": "96kl099bnr4f",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T04:41:37.652-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "The provider is currently experiencing an outage unrelated to Harness infrastructure and is investigating it. We are actively monitoring the provider’s status and will provide an update when the service is restored.",
"created_at": "2026-11-03T03:09:16.895-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T03:09:16.000-08:00",
"id": "jpp1c7ydfj6t",
"incident_id": "96kl099bnr4f",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T03:21:25.221-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-11-02T19:48:56.566-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-02T19:48:56.566-08:00",
"id": "hvnh0tgx97cf",
"incident_id": "96kl099bnr4f",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-02T19:48:56.566-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "developer.harness.io is currently degraded or unavailable due to an issue with our external hosting provider. This is outside Harness infrastructure. We're monitoring the provider's status and will update when service is restored.",
"created_at": "2026-11-02T17:02:40.852-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-02T17:02:40.852-08:00",
"id": "9p50pmjby7fm",
"incident_id": "96kl099bnr4f",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-02T17:02:40.852-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We are currently investigating this issue.",
"created_at": "2026-11-02T17:01:23.075-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-02T17:01:23.075-08:00",
"id": "hjrd36k18ltm",
"incident_id": "96kl099bnr4f",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-02T17:01:23.075-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-02T19:48:56.566-08:00",
"name": "developer.harness.io not responding to HTTPS requests",
"page_id": "zmnt6tkys0q0",
"postmortem_body": ".",
"postmortem_body_last_updated_at": "2026-11-16T11:26:07.099-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-16T11:26:07.099-08:00",
"reminder_intervals": null,
"resolved_at": "2026-11-03T18:33:11.000-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/y8g2t5wrx3j1",
"started_at": "2026-11-03T03:09:16.000-08:00",
"status": "resolved",
"updated_at": "2026-11-16T11:26:07.123-08:00"
},
{
"components": [
{
"created_at": "2024-07-10T21:10:55.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 4",
"group_id": "x1089mr22815",
"id": "czhk96jtx93b",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.784-07:00"
},
{
"created_at": "2022-03-15T18:09:19.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 3",
"group_id": "z7txn2324j6k",
"id": "4fpjqbv7k6fp",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2021-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.833-07:00"
},
{
"created_at": "2021-06-01T14:48:02.147-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 1",
"group_id": "27qq03xf2msg",
"id": "kzjygtpl015t",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-04-10T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.883-07:00"
},
{
"created_at": "2021-06-01T14:56:05.591-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "hp77w1mwh2xg",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-04-10T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.931-07:00"
}
],
"created_at": "2026-10-31T20:43:11.430-07:00",
"id": "bmyd74x3h4jy",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-10-31T22:34:51.838-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T22:34:51.838-07:00",
"id": "xr4f3c1x9fnp",
"incident_id": "bmyd74x3h4jy",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T22:34:51.838-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "Verification is currently underway for the maintenance items.",
"created_at": "2026-10-31T22:27:44.998-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T22:27:44.998-07:00",
"id": "s7r3prb66wx0",
"incident_id": "bmyd74x3h4jy",
"status": "verifying",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T22:27:44.998-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-10-31T21:47:55.852-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T21:47:55.852-07:00",
"id": "2fcwxnkqjs47",
"incident_id": "bmyd74x3h4jy",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T21:47:55.852-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Hello,\n\nWe will perform scheduled maintenance on Hosted Mac Builds to address eMudhra Certificate Authority (CA) distrust updates.\n\nImpact: Hosted Mac Builds will be unavailable during the maintenance window across all production clusters.\n\nPlease plan your builds accordingly. If you have questions or need assistance validating your pipelines, contact Harness Support.",
"created_at": "2026-10-31T20:43:11.524-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T20:43:11.524-07:00",
"id": "mg1f4pjzgt1h",
"incident_id": "bmyd74x3h4jy",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T20:43:11.524-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-31T22:27:44.998-07:00",
"name": "Hosted Mac Builds",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-31T22:34:51.838-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": "2026-10-31T21:30:00.000-07:00",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-31T23:30:00.000-07:00",
"shortlink": "https://stspg.io/sk86ygzg3fng",
"started_at": "2026-10-31T20:43:11.408-07:00",
"status": "completed",
"updated_at": "2026-10-31T22:34:51.884-07:00"
},
{
"components": [],
"created_at": "2026-10-28T10:10:38.000-07:00",
"id": "7kbnml9dyfd0",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "Starting 10/28/2026 10:10 AM CI jobs on internal VM runners intermittently hung or expired without logs, affecting some customers in prod1/2. The resolution is to upgrade the new runner version https://github.com/drone-runners/drone-runner-aws/releases/tag/v1.0.0-rc.318 (Docker image: drone/drone-runner-aws:1.0.0-rc.318)",
"created_at": "2026-10-28T12:22:38.265-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-28T10:10:38.000-07:00",
"id": "tk6kd6t940cv",
"incident_id": "7kbnml9dyfd0",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T12:23:42.184-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Intermittent CI Pipeline using VM Runner slowness and errors",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "# 1. Summary\n\nFrom October 28, 2026, 10:14 AM PT to October 29, 2026, 9:12 PM PT, CI pipelines using VM-based runners intermittently hung or expired after initialization, often with no step logs or console output. The issue affected two Prod1 customers, with additional VM-runner customers identified for proactive follow-up across Prod1, Prod2, and Prod3. The trigger was message-queue backpressure caused by broker slowness following an infrastructure image upgrade in that layer; we mitigated by rolling back that change, which resolved the platform-side trigger within about two hours. Follow-up found the two affected customers were running older versions of our VM runner and its companion log-streaming component that predated a fix for a related deadlock issue, so the customer-facing resolution was to pin to a specific, validated runner version instead of relying on a floating \"latest\" tag.\n\n# 2. Impact\n\n* CI jobs on VM runners could hang mid-run, time out after long waits , or expire without usable logs.\n* This blocked deployments for affected pipelines and slowed diagnosis, since failures were often silent\n* Two Prod1 customers were confirmed affected; a broader set of VM-runner customers across Prod1, Prod2, and Prod3 were identified as potentially exposed and proactively contacted.\n* Impact was limited to CI workflows using VM-based runners; other execution types were not affected.\n\n# 3. Root Cause\n\nOur CI VM-runner architecture streams step logs through a live-logging pipeline backed by a message-queue layer. During this incident, that queue experienced backpressure, following an infrastructure image upgrade on one of the queue's brokers. This degraded the live-log streaming path platform-wide, causing some in-flight log writes to stall.\n\n\n\nHowever, this platform-level degradation was transient and self-healed once we rolled back the image upgrade. It only turned into multi-hour, customer-visible hangs for customers who were running older builds of our VM runner and its companion log-streaming helper component,specifically, builds that predated a fix for a known deadlock condition in how that component handled interrupted log streams. Under normal queue conditions, this defect was dormant; the queue backpressure was enough to trigger it on those older builds, causing their executions to hang on an internal request until a roughly 4-hour timeout expired.\n\nBoth confirmed affected customers were using runner versions older than the version containing the fix, and in one case the customer believed they were on the latest available image but were, in practice, still running an older pulled version — because our \"latest\" tag at the time did not reliably correspond to the newest, QA-validated build.\n\n# 4. Resolution\n\n* Identified that failures were specific to VM-runner executions hanging after initialization, and separated this from an unrelated initialization-stage incident from the prior day.\n* Traced message-queue producer errors and broker slowness to a recent infrastructure image upgrade on that queue layer.\n* Rolled back that image upgrade, which brought queue/log-streaming error rates to zero within about two hours of incident start.\n* Confirmed, with the two affected customers, the specific runner and log-streaming component versions they were running.\n* Recommended and confirmed a specific, pinned runner version containing the deadlock fix as the customer-facing resolution.\n* Identified additional VM-runner customers for proactive outreach and validation across Prod1, Prod2, and Prod3.\n\n# 5. Preventive Actions\n\n1. **Surface the VM runner version** directly in pipeline execution logs and metadata, so this no longer has to be manually confirmed with customers during an incident.\n2. **Publish guidance** recommending customers pin to an explicit runner version rather than relying on \"latest\".\n3. **Add detection for post-initialization stalls**, specifically keyed to the ~4-hour timeout signature seen in this incident.\n4. **Build a queryable inventory** of VM-runner customers and the versions they're running, to support faster, more targeted response in future incidents and proactive upgrade outreach.",
"postmortem_body_last_updated_at": "2026-11-10T15:02:53.163-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-10T15:02:53.163-08:00",
"reminder_intervals": null,
"resolved_at": "2026-10-28T10:10:38.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/fbvsnn07z92v",
"started_at": "2026-10-28T10:10:38.000-07:00",
"status": "resolved",
"updated_at": "2026-11-10T15:02:53.211-08:00"
},
{
"components": [
{
"created_at": "2023-08-21T10:57:32.631-07:00",
"description": "Harness Continuous Integration Enterprise (CIE) - Self Hosted Runners in Prod 1",
"group_id": "27qq03xf2msg",
"id": "cc654c3j6476",
"name": "Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2023-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-27T15:24:17.476-07:00"
}
],
"created_at": "2026-10-27T12:52:30.529-07:00",
"id": "jmfpjjljc0n3",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-27T15:24:17.519-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T15:24:17.519-07:00",
"id": "nfvqchtxzw13",
"incident_id": "jmfpjjljc0n3",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T15:24:17.519-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-27T15:16:36.592-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T15:16:36.592-07:00",
"id": "wkgl2c4njw6z",
"incident_id": "jmfpjjljc0n3",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T15:16:36.592-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-27T14:15:10.997-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T14:15:10.997-07:00",
"id": "vfk9lx3m5ms0",
"incident_id": "jmfpjjljc0n3",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T14:15:10.997-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "CI pipelines on VM infrastructure are failing at the Initialize step because the task goes to a delegate outside the intended pool",
"created_at": "2026-10-27T12:58:34.228-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T12:58:34.228-07:00",
"id": "x2zv7cmhktrq",
"incident_id": "jmfpjjljc0n3",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T12:58:34.228-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-27T12:52:30.648-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T12:52:30.648-07:00",
"id": "jzgvzwlfdlvm",
"incident_id": "jmfpjjljc0n3",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T12:52:30.648-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-27T15:16:36.592-07:00",
"name": "We are currently seeing delegate selection issue for VM infra executions",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "## 1. Summary\n\nOn October 27, 2026, starting around 6:35 AM, customers running CI pipelines on self-managed VM infrastructure \\(a legacy Delegate-based VM runner configuration\\) in Prod1 began experiencing pipeline failures caused by tasks being routed to Delegates that could not actually execute them. The issue affected a small number of customers using this specific, less common configuration \\(VM-based CI without Delegate tags configured\\). It was introduced by a backend deployment and was mitigated by rolling that deployment back; a permanent fix followed.\n\n# 2. Impact\n\n* CI pipelines using VM-based execution without Delegate tags configured began intermittently failing to run on affected Delegates.\n* Other pipeline types and configurations were not affected.\n* Impact was limited to a small number of customers in Prod1 using this specific configuration.\n* The partial outage lasted approximately 8 hours before being resolved via rollback. Affected customers were given a configuration-based workaround \\(setting Delegate tags\\) during the incident.\n\n# 3. Root Cause\n\nOur Delegate platform uses a feature called Capability Checks to determine whether a given Delegate is actually able to execute a specific task, for example, whether it has access to the right infrastructure or connector. When Delegate tags are explicitly configured, this check matters less, but for configurations without tags, it is the primary mechanism for routing work correctly.\n\n\n\nAs part of ongoing work to support capability checking for a newer Delegate version, an optimization was introduced that skipped running a capability check whenever its underlying validation criteria came back empty, the assumption being that an empty criteria value meant there was nothing meaningful to validate, so the Delegate could be treated as already capable.\n\n\n\nIn practice, several existing capability checks had previously harmless defect, which under certain conditions would produce an empty criteria value despite still having a meaningful check that should run. Skipping checks in that state caused Delegates to be automatically treated as capable regardless of whether they actually were and most significantly, for legacy CI VM pipelines, where this logic determines whether a given Delegate is managing the correct virtual machine pool. With the check skipped, Delegates were marked capable of managing any VM pool, which caused CI tasks to be routed to Delegates that could not actually run them.\n\n\n\nBecause the underlying defects were pre-existing but previously harmless, the issue only surfaced once the new optimization began skipping the checks that had been quietly masking them, and it took several hours to appear widely because of a caching layer with a 6-hour refresh interval.\n\n# 4. Resolution\n\n* Identified that CI VM task routing was being driven by faulty capability-check results.\n* Rolled back the backend deployment containing the optimization \\(the first rollback attempt did not fully take effect; a retry succeeded\\).\n* Provided customers with a configuration-based workaround \\(explicitly setting Delegate tags\\) to reduce impact while the rollback was in progress.\n* Reverted the capability-check optimization so checks with empty criteria run normally again rather than being skipped.\n* Added metrics and logging to help identify the specific capability checks that were producing empty criteria incorrectly, so they can be fixed at the source.\n\n# 5. Preventive Actions\n\n1. **Fix the underlying capability checks** that were producing empty criteria incorrectly \\(several affecting CI VM pipelines, with a few additional minor cases identified\\).\n2. **Safely re-introduce the capability-check optimization** once those checks are fixed, since empty criteria should not normally occur once the underlying defects are resolved.\n3. **Add automated test coverage** that exercises this CI VM Delegate-routing use case, since this was a regression that automated tests did not catch.\n4. **Add alerting specific to CI VM infrastructure routing**, since no automated alert fired for this incident and it was first identified by customers.",
"postmortem_body_last_updated_at": "2026-11-10T14:55:40.009-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-10T14:55:40.009-08:00",
"reminder_intervals": null,
"resolved_at": "2026-10-27T15:24:17.519-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/xsr0cq7g8zjm",
"started_at": "2026-10-27T12:52:30.520-07:00",
"status": "resolved",
"updated_at": "2026-11-10T14:55:40.039-08:00"
},
{
"components": [
{
"created_at": "2024-11-07T20:37:07.688-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "rgdxklp8p0kd",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.261-07:00"
},
{
"created_at": "2024-11-07T20:34:14.643-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "pb161tg7ccrs",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.306-07:00"
},
{
"created_at": "2024-11-07T20:38:19.182-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "0kkbcr1h3cvx",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.350-07:00"
},
{
"created_at": "2024-11-07T20:40:44.997-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "trkh53c2bscy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.391-07:00"
},
{
"created_at": "2024-11-07T20:32:37.209-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 1",
"group_id": "27qq03xf2msg",
"id": "n0m4jk53hrym",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.435-07:00"
},
{
"created_at": "2024-11-07T20:33:35.371-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "ctmk1q1dw7nr",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.484-07:00"
},
{
"created_at": "2024-11-07T20:40:12.699-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "th62w3qy4r3m",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.526-07:00"
},
{
"created_at": "2024-11-07T20:41:13.208-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "x3wy7hhj591c",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.579-07:00"
},
{
"created_at": "2026-07-22T22:27:49.688-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 6",
"group_id": "1fpldmj5f61k",
"id": "t8gybmf0b2qt",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.715-07:00"
},
{
"created_at": "2024-11-07T20:39:17.107-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod1",
"group_id": "27qq03xf2msg",
"id": "5znhlv1nwfcf",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.857-07:00"
},
{
"created_at": "2024-11-07T20:39:41.158-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod2",
"group_id": "qbxn0mh45jhv",
"id": "qvs27pfch8vy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.900-07:00"
},
{
"created_at": "2026-07-22T22:28:26.126-07:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod6",
"group_id": "1fpldmj5f61k",
"id": "q1369hjq7nn4",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.939-07:00"
}
],
"created_at": "2026-10-19T02:19:47.725-07:00",
"id": "9734l1pk9khw",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-10-23T23:32:06.425-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T23:32:06.425-07:00",
"id": "gw732td06qkl",
"incident_id": "9734l1pk9khw",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T23:32:06.425-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "Verification is currently underway for the maintenance items.",
"created_at": "2026-10-23T23:20:49.005-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T23:20:49.005-07:00",
"id": "yv6bvddnf3wl",
"incident_id": "9734l1pk9khw",
"status": "verifying",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T23:20:49.005-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-10-23T22:36:51.565-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T22:36:51.565-07:00",
"id": "xdsh4q3vk6v7",
"incident_id": "9734l1pk9khw",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T22:36:51.565-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We're upgrading our CI infrastructure to improve reliability and security.\n\n Expected impact:\n - Brief connectivity blips as we migrate outbound traffic.\n - Builds running during the migration may fail on outbound calls\n (git clone, artifact pulls, webhooks) and will need a re-run.\n - Re-runs will succeed once the migration completes.\n\n Your action:\n If you haven't already allowlisted the new egress IP ranges\n (8.235.0.0/24 and 34.33.12.0/24), please do so now.",
"created_at": "2026-10-19T02:19:47.855-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T02:19:47.855-07:00",
"id": "s5fw2kdhw7h7",
"incident_id": "9734l1pk9khw",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T02:19:47.855-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-23T23:20:49.005-07:00",
"name": "Planned maintenance: Harness CI egress IP migration",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-23T23:32:06.425-07:00",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-24T22:30:00.000-07:00",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-24T23:30:00.000-07:00",
"shortlink": "https://stspg.io/6sm9tqxjs3sl",
"started_at": "2026-10-19T02:19:47.674-07:00",
"status": "completed",
"updated_at": "2026-10-23T23:32:06.470-07:00"
},
{
"components": [
{
"created_at": "2024-07-10T21:08:30.607-07:00",
"description": "Global Gateway intermittently encountering connection errors in Prod 4",
"group_id": "x1089mr22815",
"id": "p9t9g01vgy47",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 1,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-11-11T22:07:00.552-08:00"
},
{
"created_at": "2024-07-10T21:11:25.979-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Self Hosted Runners in Prod 4",
"group_id": "x1089mr22815",
"id": "bp1hsfllmds0",
"name": "Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"page_id": "zmnt6tkys0q0",
"position": 4,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.069-07:00"
},
{
"created_at": "2024-07-10T21:10:55.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 4",
"group_id": "x1089mr22815",
"id": "czhk96jtx93b",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.784-07:00"
},
{
"created_at": "2024-11-07T20:37:07.688-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "rgdxklp8p0kd",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.261-07:00"
},
{
"created_at": "2024-11-07T20:40:44.997-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "trkh53c2bscy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.391-07:00"
},
{
"created_at": "2024-07-10T21:12:38.445-07:00",
"description": "Harness Security Testing Orchestration (STO) in Prod 4",
"group_id": "x1089mr22815",
"id": "k0hbqz5c81v8",
"name": "Security Testing Orchestration (STO)",
"page_id": "zmnt6tkys0q0",
"position": 10,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.164-07:00"
},
{
"created_at": "2024-07-10T21:20:08.616-07:00",
"description": "Harness Infrastructure as Code Management (IaCM) in Prod 4",
"group_id": "x1089mr22815",
"id": "nxsthnf9gk3g",
"name": "Infrastructure as Code Management (IaCM)",
"page_id": "zmnt6tkys0q0",
"position": 14,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.188-07:00"
},
{
"created_at": "2025-08-27T10:29:44.922-07:00",
"description": "Core platform constructs eg. Platform delegate (eg. Delegate communication), Platform Core (eg. Account, Login, Project/Org, Connectors, Secrets, Delegate, Settings, Notifications, Audits, Templates, Services, Environments , Policies, File Store, Log Uploads), Platform Authentication \u0026 Authorization (login, RBAC, logout, API keys, Service account, Tokens etc.)",
"group_id": "x1089mr22815",
"id": "561g9b7sqgct",
"name": "Platform",
"page_id": "zmnt6tkys0q0",
"position": 17,
"showcase": true,
"start_date": "2025-08-27T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.212-07:00"
}
],
"created_at": "2026-10-19T00:46:51.821-07:00",
"id": "9xpkprn54m6z",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-19T01:21:25.895-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:21:25.895-07:00",
"id": "tgzqsf2538kw",
"incident_id": "9xpkprn54m6z",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:21:25.895-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-19T01:04:11.251-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:04:11.251-07:00",
"id": "912js2k93ds5",
"incident_id": "9xpkprn54m6z",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:04:11.251-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-19T01:02:50.060-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:02:50.060-07:00",
"id": "3zwtk30334mw",
"incident_id": "9xpkprn54m6z",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:02:50.060-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-19T01:02:29.637-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:02:29.637-07:00",
"id": "jj7zlhzsgn6h",
"incident_id": "9xpkprn54m6z",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:02:29.637-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-19T00:46:52.239-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T00:46:52.239-07:00",
"id": "j508529yg8js",
"incident_id": "9xpkprn54m6z",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T00:46:52.239-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-19T01:02:29.637-07:00",
"name": "Degraded performance on prod4",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "## **Summary**\n\nOn October 19, 2026, between approximately 12:10 AM and 12:55 AM PDT, customers on Prod4 experienced errors and intermittent unavailability of Harness Pipelines services. Heavy pipeline executions drove a sharp rise in CPU and memory use on the pipeline service, and for a short period all service instances were unavailable. Harness engineering scaled the service both vertically and horizontally, which restored service. A fix for the underlying behavior is being rolled out.\n\n## **Root Cause**\n\nOn October 19th, high number of concurrent executions with very large execution graphs drove a sharp increase in cluster resources. The service stores and retrieves this full execution graph for visualization purposes and further progressing the pipelines to next states. Our auto-scalaers on Prod4 took longer than usual for scaling resources due to a sharp rise in CPU and memory. \n\nAs a part of the fix, the execution graph update will now process graph changes incrementally instead of the full graph. It is being rolled out in a phased manner and will be completely rolled out at the earliest.\n\n## **Impact**\n\n* Affected users: Customers on Prod4.\n* Symptom: unavailability of pipeline functionality in the Harness platform.\n* Data: No data loss.\n* Suggested workaround: None required from customers. Harness engineering resolved the issue.\n* Duration of impact: Approximately 20 minutes of full unavailability and degraded performance for 25 minutes. Harness continued monitoring after recovery and confirmed platform stability with customers.\n\n## **Remediation**\n\n**Immediate**\n\nHarness engineering scaled the pipeline service both vertically \\(more resources per instance\\) and horizontally \\(more instances\\), and removed instances in an unhealthy state. The instance count stabilized and normal service was restored. The system will stay scaled up till the fix is completely rolled out. \n\n**Permanent fix and Action Items**\n\nTo prevent such issues from happening again, Harness is\n\n1. Tuning the auto heal/scaling process so that impact is mimimal to none.\n2. Optimizing the execution graph flow so that large graphs can be process faster.",
"postmortem_body_last_updated_at": "2026-10-23T11:24:10.455-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-23T11:24:10.455-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-19T01:21:25.895-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/bmvp2852wlrr",
"started_at": "2026-10-19T00:46:51.798-07:00",
"status": "resolved",
"updated_at": "2026-10-23T11:24:10.479-07:00"
}
],
"page": {
"id": "zmnt6tkys0q0",
"name": "Harness",
"url": "https://status.harness.io"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}