Snapshot 149054
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"nxbqtxqrm55f",
"xs5vhwly9mk5",
"f347vm5yn95f",
"vxl2rz20dcyr",
"yyv8hjtmvpzv",
"cc654c3j6476",
"kzjygtpl015t",
"n0m4jk53hrym",
"5znhlv1nwfcf",
"83n5xv3hpb69",
"by80qyspb2jq",
"sn84zlkwcg24",
"lv49p6nbl193",
"5qd852klgxc0",
"15bv7h1knvsk",
"mn3934kz3w69",
"5rys3t6403zn",
"1lccfpfwmh4y",
"rz6k0bcnrpr1",
"vy68z1vstc62",
"l539c2tfydzj",
"8vf2zpmhfyy0",
"j4tq5500kmg8",
"77bh7sgmbbv3"
],
"created_at": "2021-06-01T14:46:03.030-07:00",
"description": null,
"group_id": null,
"id": "27qq03xf2msg",
"name": "Prod 1",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.090-07:00"
},
{
"components": [
"s54kkcr6t0gq",
"gbltvmpdkb3r",
"bqhhx747whp3",
"tgcylzz5nlry",
"c6myzvwng2lw",
"9677ldz5y7sd",
"hp77w1mwh2xg",
"ctmk1q1dw7nr",
"qvs27pfch8vy",
"65kdr9mrlblc",
"cg2xjl1jzk0m",
"qbzy2sc4skqh",
"gdlfxhr31qmb",
"sbwzykrf1q8t",
"b4tvy5s9wr6z",
"v1slhlqmmbm1",
"5k3yywp3xvqj",
"640z11cmjv3n",
"fwh6z4glx958",
"0yrdl70cplw9",
"czfjkrnqlcsk",
"9pr1l3rwf14n",
"8tnc91hlt8d2"
],
"created_at": "2021-06-01T14:53:31.712-07:00",
"description": null,
"group_id": null,
"id": "qbxn0mh45jhv",
"name": "Prod 2",
"page_id": "zmnt6tkys0q0",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.103-07:00"
},
{
"components": [
"rwx7y5zv4yq5",
"l1sx8bck1yzb",
"fw0zmk94bkpc",
"tq8rdbq9np4d",
"brmlw2942fbc",
"4fpjqbv7k6fp",
"pb161tg7ccrs",
"th62w3qy4r3m",
"kxxh18chwqlg",
"k4sqhp5t769d",
"nw83hg1v0ftp",
"q489v85zw6qs",
"vf6qxw4l52m4",
"n0f6mg62hgx3",
"ths5wq122vtx",
"804lzkqf2vfk",
"jjmrjld2vwnb",
"yhy2w4n4q7kk",
"sbm4gmjj60nl",
"g5xm7jlw5rr4",
"ztrdyp8d48sx",
"mbpd7mcgc0l0"
],
"created_at": "2021-06-01T14:54:00.058-07:00",
"description": null,
"group_id": null,
"id": "z7txn2324j6k",
"name": "Prod 3",
"page_id": "zmnt6tkys0q0",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.115-07:00"
},
{
"components": [
"p9t9g01vgy47",
"5161jzbx0d8w",
"j9qkdrxdylw2",
"bp1hsfllmds0",
"czhk96jtx93b",
"rgdxklp8p0kd",
"trkh53c2bscy",
"sfc57k1p0sqp",
"790nrgxflg5x",
"k0hbqz5c81v8",
"h1wfypqglf49",
"gvhsp1j79nbm",
"xzg7lrrqyk37",
"nxsthnf9gk3g",
"6xhczbg7xfyv",
"0vlv5pjpjzbs",
"561g9b7sqgct"
],
"created_at": "2024-07-10T21:08:30.585-07:00",
"description": null,
"group_id": null,
"id": "x1089mr22815",
"name": "Prod 4",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.126-07:00"
},
{
"components": [
"zbyg84df96x7",
"007h1ztzxtlw",
"v7mz6ltk413q",
"w3csxhc0mqz7",
"rbn7hjb2qf8r",
"y2wg26xx8r7r",
"0kkbcr1h3cvx",
"x3wy7hhj591c",
"z9267hzwpnwq",
"fwvncn7t50n9",
"r87dg7ccrjpv",
"lg2vtjgxqkzw",
"y3rkhygkv1g3",
"1b70wzgp8spj",
"3hdhfjy41l7p",
"rh8y3gywr698",
"zp7rk1rtt25b",
"yhy0vm97y09n"
],
"created_at": "2024-09-09T21:49:39.223-07:00",
"description": null,
"group_id": null,
"id": "6y3np02bqtxf",
"name": "Prod Eu 1",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-22T04:49:45.778-07:00"
},
{
"created_at": "2021-12-10T16:29:47.633-08:00",
"description": "Harness OverOps",
"group_id": null,
"id": "1h5qbtxd2622",
"name": "Service Reliability Management - Error Tracking FirstGen (fka OverOps)",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-12-10T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2024-10-28T19:09:00.147-07:00"
},
{
"components": [
"52zt8q9l5yk4",
"wjt25yrgfl1d"
],
"created_at": "2025-01-13T08:34:11.290-08:00",
"description": null,
"group_id": null,
"id": "rx8vt1qyrpsv",
"name": "Software Engineering Insights FirstGen (fka Propelo)",
"page_id": "zmnt6tkys0q0",
"position": 10,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-01-13T08:34:11.290-08:00"
},
{
"components": [
"hnbf7sq0cq5b",
"s0dh2g03zz7r",
"fbbt3z4ytn23",
"yxvnmfxvggb9",
"0r0gmbxty2d9",
"dnkfpsn0r5vr"
],
"created_at": "2025-09-01T23:44:00.402-07:00",
"description": "Traceable Clusters",
"group_id": null,
"id": "np782snlmj00",
"name": "Traceable",
"page_id": "zmnt6tkys0q0",
"position": 11,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-09-01T23:44:29.556-07:00"
},
{
"components": [
"qzlxgjbrdby4",
"gflflybxx6b8",
"zwf5f0yg33qk",
"w90818668zwc",
"xwbxwtcl9g25",
"y21hwc1hhpnz",
"qcj53m9f1ptq",
"6lmbj16bvn16"
],
"created_at": "2025-11-05T10:12:41.367-08:00",
"description": "Feature Management \u0026 Experimentation",
"group_id": null,
"id": "wm26z10lksx3",
"name": "Feature Management \u0026 Experimentation (FME)",
"page_id": "zmnt6tkys0q0",
"position": 13,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-02-04T14:36:46.108-08:00"
},
{
"components": [
"s1dlyyndwjl6",
"6k06w76wqwsc",
"p20hnvtnpw1b",
"drq1sbn8hw11",
"v5rf21nbqwg2"
],
"created_at": "2026-01-06T02:08:28.178-08:00",
"description": "Dedicated SAAS cluster for Goldman Sachs",
"group_id": null,
"id": "2w5qsvz2lk4n",
"name": "Prod 7",
"page_id": "zmnt6tkys0q0",
"position": 14,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-01-06T02:11:59.702-08:00"
},
{
"components": [
"0rxrdlk4dr3m",
"snxfjcdrnddd",
"lfrwy26bjy27",
"s9f6pwt8tbj1",
"gl4wwd0s6l4n",
"7jfnyljgkhsc"
],
"created_at": "2026-01-06T02:35:45.999-08:00",
"description": "Dedicated SAAS cluster for Barclays",
"group_id": null,
"id": "n6rk38cphg56",
"name": "Prod 8",
"page_id": "zmnt6tkys0q0",
"position": 15,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-01-06T02:36:31.092-08:00"
},
{
"components": [
"shg48w7t35l8",
"1zmt4tlpxh7d",
"tsvcht7ggxpq",
"6q2y478qnshb",
"5vjk880lv3yb",
"vqy2pfsw7sxx",
"gfsntx0t1n45",
"glxwgv0pdjrb",
"6cq3s06svg8n",
"d7b0scq7bvf5",
"xckv4nnb4hjy",
"t9pvbwy0smfp",
"m25krg585c3z",
"7fp2gfxytcw0",
"l5np7vy2h0bc",
"kbjf89kfnxfm",
"z1r28087zxf5",
"gm8yq9pr4xbk",
"47jx56p992v7",
"j4mnhf18ky22",
"93j99d3r0nb3",
"mttlsgnqy0mw",
"1yjpjqbtb1xn"
],
"created_at": "2026-04-14T01:59:10.412-07:00",
"description": null,
"group_id": null,
"id": "j9f15dxpqydw",
"name": "Dev",
"page_id": "zmnt6tkys0q0",
"position": 16,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-11T07:30:50.641-07:00"
},
{
"components": [
"tgbvl8jm69gd",
"n48l3qtwftyq",
"9nh0xfkz9hkq",
"jddlj6k9xs29"
],
"created_at": "2026-04-20T02:54:03.635-07:00",
"description": null,
"group_id": null,
"id": "23r8514jn030",
"name": "Prod 5",
"page_id": "zmnt6tkys0q0",
"position": 17,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-04-20T02:55:08.885-07:00"
},
{
"created_at": "2026-06-19T02:52:03.505-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Dev",
"group_id": null,
"id": "9r4nnf9jzl4t",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-06-19T02:52:03.505-07:00"
},
{
"components": [
"y41n6v53x5z1",
"hbt1g5bqvg1l",
"hjr9dqb9t38t",
"th0ygdtzlvdk",
"t45ncxygrpxm",
"nyhhbjtgtst4",
"8mjkl07z9kn1",
"t8gybmf0b2qt",
"q1369hjq7nn4",
"60cy4p5vr6gp",
"xbqdy20jrrs5",
"skccvt241051",
"52ny1ymph3wd",
"2cn1374658lx",
"r09j2jvg6r3h",
"l0fd5d66jnpp",
"kryxb6g45sqn",
"lbdf8v2kkq2j",
"hg36cdlm95bl",
"fhb7jrwcfdzl",
"w9kcl4xznsym",
"jdc2stt9jvds",
"kf7xlcfh77bx"
],
"created_at": "2026-07-22T22:21:45.295-07:00",
"description": null,
"group_id": null,
"id": "1fpldmj5f61k",
"name": "Prod 6",
"page_id": "zmnt6tkys0q0",
"position": 19,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-22T22:21:45.295-07:00"
}
],
"incidents": [
{
"components": [
{
"created_at": "2024-11-07T20:37:07.688-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "rgdxklp8p0kd",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.261-07:00"
},
{
"created_at": "2024-11-07T20:34:14.643-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "pb161tg7ccrs",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.306-07:00"
},
{
"created_at": "2024-11-07T20:38:19.182-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "0kkbcr1h3cvx",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.350-07:00"
},
{
"created_at": "2024-11-07T20:40:44.997-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "trkh53c2bscy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.391-07:00"
},
{
"created_at": "2024-11-07T20:32:37.209-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 1",
"group_id": "27qq03xf2msg",
"id": "n0m4jk53hrym",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.435-07:00"
},
{
"created_at": "2024-11-07T20:33:35.371-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "ctmk1q1dw7nr",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.484-07:00"
},
{
"created_at": "2024-11-07T20:40:12.699-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "th62w3qy4r3m",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.526-07:00"
},
{
"created_at": "2024-11-07T20:41:13.208-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "x3wy7hhj591c",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.579-07:00"
},
{
"created_at": "2026-07-22T22:27:49.688-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 6",
"group_id": "1fpldmj5f61k",
"id": "t8gybmf0b2qt",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.715-07:00"
},
{
"created_at": "2024-11-07T20:39:17.107-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod1",
"group_id": "27qq03xf2msg",
"id": "5znhlv1nwfcf",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.857-07:00"
},
{
"created_at": "2024-11-07T20:39:41.158-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod2",
"group_id": "qbxn0mh45jhv",
"id": "qvs27pfch8vy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.900-07:00"
},
{
"created_at": "2026-07-22T22:28:26.126-07:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod6",
"group_id": "1fpldmj5f61k",
"id": "q1369hjq7nn4",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.939-07:00"
}
],
"created_at": "2026-10-19T02:19:47.725-07:00",
"id": "9734l1pk9khw",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-10-23T23:32:06.425-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T23:32:06.425-07:00",
"id": "gw732td06qkl",
"incident_id": "9734l1pk9khw",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T23:32:06.425-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "Verification is currently underway for the maintenance items.",
"created_at": "2026-10-23T23:20:49.005-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T23:20:49.005-07:00",
"id": "yv6bvddnf3wl",
"incident_id": "9734l1pk9khw",
"status": "verifying",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T23:20:49.005-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-10-23T22:36:51.565-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T22:36:51.565-07:00",
"id": "xdsh4q3vk6v7",
"incident_id": "9734l1pk9khw",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T22:36:51.565-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We're upgrading our CI infrastructure to improve reliability and security.\n\n Expected impact:\n - Brief connectivity blips as we migrate outbound traffic.\n - Builds running during the migration may fail on outbound calls\n (git clone, artifact pulls, webhooks) and will need a re-run.\n - Re-runs will succeed once the migration completes.\n\n Your action:\n If you haven't already allowlisted the new egress IP ranges\n (8.235.0.0/24 and 34.33.12.0/24), please do so now.",
"created_at": "2026-10-19T02:19:47.855-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T02:19:47.855-07:00",
"id": "s5fw2kdhw7h7",
"incident_id": "9734l1pk9khw",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T02:19:47.855-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-23T23:20:49.005-07:00",
"name": "Planned maintenance: Harness CI egress IP migration",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-23T23:32:06.425-07:00",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-24T22:30:00.000-07:00",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-24T23:30:00.000-07:00",
"shortlink": "https://stspg.io/6sm9tqxjs3sl",
"started_at": "2026-10-19T02:19:47.674-07:00",
"status": "completed",
"updated_at": "2026-10-23T23:32:06.470-07:00"
},
{
"components": [
{
"created_at": "2024-07-10T21:08:30.607-07:00",
"description": "Global Gateway intermittently encountering connection errors in Prod 4",
"group_id": "x1089mr22815",
"id": "p9t9g01vgy47",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 1,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.045-07:00"
},
{
"created_at": "2024-07-10T21:11:25.979-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Self Hosted Runners in Prod 4",
"group_id": "x1089mr22815",
"id": "bp1hsfllmds0",
"name": "Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"page_id": "zmnt6tkys0q0",
"position": 4,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.069-07:00"
},
{
"created_at": "2024-07-10T21:10:55.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 4",
"group_id": "x1089mr22815",
"id": "czhk96jtx93b",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.091-07:00"
},
{
"created_at": "2024-11-07T20:37:07.688-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "rgdxklp8p0kd",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.261-07:00"
},
{
"created_at": "2024-11-07T20:40:44.997-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "trkh53c2bscy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.391-07:00"
},
{
"created_at": "2024-07-10T21:12:38.445-07:00",
"description": "Harness Security Testing Orchestration (STO) in Prod 4",
"group_id": "x1089mr22815",
"id": "k0hbqz5c81v8",
"name": "Security Testing Orchestration (STO)",
"page_id": "zmnt6tkys0q0",
"position": 10,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.164-07:00"
},
{
"created_at": "2024-07-10T21:20:08.616-07:00",
"description": "Harness Infrastructure as Code Management (IaCM) in Prod 4",
"group_id": "x1089mr22815",
"id": "nxsthnf9gk3g",
"name": "Infrastructure as Code Management (IaCM)",
"page_id": "zmnt6tkys0q0",
"position": 14,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.188-07:00"
},
{
"created_at": "2025-08-27T10:29:44.922-07:00",
"description": "Core platform constructs eg. Platform delegate (eg. Delegate communication), Platform Core (eg. Account, Login, Project/Org, Connectors, Secrets, Delegate, Settings, Notifications, Audits, Templates, Services, Environments , Policies, File Store, Log Uploads), Platform Authentication \u0026 Authorization (login, RBAC, logout, API keys, Service account, Tokens etc.)",
"group_id": "x1089mr22815",
"id": "561g9b7sqgct",
"name": "Platform",
"page_id": "zmnt6tkys0q0",
"position": 17,
"showcase": true,
"start_date": "2025-08-27T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.212-07:00"
}
],
"created_at": "2026-10-19T00:46:51.821-07:00",
"id": "9xpkprn54m6z",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-19T01:21:25.895-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:21:25.895-07:00",
"id": "tgzqsf2538kw",
"incident_id": "9xpkprn54m6z",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:21:25.895-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-19T01:04:11.251-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:04:11.251-07:00",
"id": "912js2k93ds5",
"incident_id": "9xpkprn54m6z",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:04:11.251-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-19T01:02:50.060-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:02:50.060-07:00",
"id": "3zwtk30334mw",
"incident_id": "9xpkprn54m6z",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:02:50.060-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-19T01:02:29.637-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:02:29.637-07:00",
"id": "jj7zlhzsgn6h",
"incident_id": "9xpkprn54m6z",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:02:29.637-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-19T00:46:52.239-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T00:46:52.239-07:00",
"id": "j508529yg8js",
"incident_id": "9xpkprn54m6z",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T00:46:52.239-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-19T01:02:29.637-07:00",
"name": "Degraded performance on prod4",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "## **Summary**\n\nOn October 19, 2026, between approximately 12:10 AM and 12:55 AM PDT, customers on Prod4 experienced errors and intermittent unavailability of Harness Pipelines services. Heavy pipeline executions drove a sharp rise in CPU and memory use on the pipeline service, and for a short period all service instances were unavailable. Harness engineering scaled the service both vertically and horizontally, which restored service. A fix for the underlying behavior is being rolled out.\n\n## **Root Cause**\n\nOn October 19th, high number of concurrent executions with very large execution graphs drove a sharp increase in cluster resources. The service stores and retrieves this full execution graph for visualization purposes and further progressing the pipelines to next states. Our auto-scalaers on Prod4 took longer than usual for scaling resources due to a sharp rise in CPU and memory. \n\nAs a part of the fix, the execution graph update will now process graph changes incrementally instead of the full graph. It is being rolled out in a phased manner and will be completely rolled out at the earliest.\n\n## **Impact**\n\n* Affected users: Customers on Prod4.\n* Symptom: unavailability of pipeline functionality in the Harness platform.\n* Data: No data loss.\n* Suggested workaround: None required from customers. Harness engineering resolved the issue.\n* Duration of impact: Approximately 20 minutes of full unavailability and degraded performance for 25 minutes. Harness continued monitoring after recovery and confirmed platform stability with customers.\n\n## **Remediation**\n\n**Immediate**\n\nHarness engineering scaled the pipeline service both vertically \\(more resources per instance\\) and horizontally \\(more instances\\), and removed instances in an unhealthy state. The instance count stabilized and normal service was restored. The system will stay scaled up till the fix is completely rolled out. \n\n**Permanent fix and Action Items**\n\nTo prevent such issues from happening again, Harness is\n\n1. Tuning the auto heal/scaling process so that impact is mimimal to none.\n2. Optimizing the execution graph flow so that large graphs can be process faster.",
"postmortem_body_last_updated_at": "2026-10-23T11:24:10.455-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-23T11:24:10.455-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-19T01:21:25.895-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/bmvp2852wlrr",
"started_at": "2026-10-19T00:46:51.798-07:00",
"status": "resolved",
"updated_at": "2026-10-23T11:24:10.479-07:00"
},
{
"components": [
{
"created_at": "2022-03-15T17:37:45.978-07:00",
"description": "Harness Continuous Delivery Next Generation (CDNG) in Prod 3",
"group_id": "z7txn2324j6k",
"id": "l1sx8bck1yzb",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": true,
"start_date": "2021-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-15T03:27:52.167-07:00"
}
],
"created_at": "2026-10-15T01:04:14.381-07:00",
"id": "xdt6vk6q7k5k",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-15T01:15:44.044-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T01:15:44.044-07:00",
"id": "df7fdxc61y5d",
"incident_id": "xdt6vk6q7k5k",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T01:15:44.044-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-15T01:09:35.314-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T01:09:35.314-07:00",
"id": "76sy7n6k44lw",
"incident_id": "xdt6vk6q7k5k",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T01:09:35.314-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-15T01:05:07.284-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T01:05:07.284-07:00",
"id": "2yyjbx0mmy2z",
"incident_id": "xdt6vk6q7k5k",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T01:05:07.284-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-15T01:04:14.652-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T01:04:14.652-07:00",
"id": "31f80hzql4y6",
"incident_id": "xdt6vk6q7k5k",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T01:04:14.652-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-15T01:09:35.314-07:00",
"name": "Degraded performance in UI execution graph in prod3",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-15T01:15:44.044-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/8q9r632ktw3c",
"started_at": "2026-10-15T01:04:14.373-07:00",
"status": "resolved",
"updated_at": "2026-10-15T01:15:44.222-07:00"
},
{
"components": [
{
"created_at": "2024-04-17T02:36:03.213-07:00",
"description": "Harness Infrastructure as Code Management (IaCM) in Prod1",
"group_id": "27qq03xf2msg",
"id": "15bv7h1knvsk",
"name": "Infrastructure as Code Management (IaCM)",
"page_id": "zmnt6tkys0q0",
"position": 15,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-12T18:56:07.012-07:00"
}
],
"created_at": "2026-10-12T18:06:13.615-07:00",
"id": "20bc41tsx8rz",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-12T21:29:35.168-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-12T21:29:35.168-07:00",
"id": "x4hf8d89h2yg",
"incident_id": "20bc41tsx8rz",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T21:29:35.168-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-12T18:56:07.210-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-12T18:56:07.210-07:00",
"id": "h98y6qvv1s6w",
"incident_id": "20bc41tsx8rz",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T18:56:07.210-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-12T18:07:25.351-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-12T18:07:25.351-07:00",
"id": "lsh49l5zy6r8",
"incident_id": "20bc41tsx8rz",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T18:07:25.351-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-12T18:06:13.722-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-12T18:06:13.722-07:00",
"id": "2gqhv0hv98xb",
"incident_id": "20bc41tsx8rz",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T18:06:13.722-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-12T18:56:07.210-07:00",
"name": "Prod1: Intermittent issue with IACM stages in some pipelines",
"page_id": "zmnt6tkys0q0",
"postmortem_body": ". \n\n.",
"postmortem_body_last_updated_at": "2026-10-22T20:16:42.348-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-22T20:16:42.348-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-12T21:29:35.168-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/zygklrx86y2b",
"started_at": "2026-10-12T18:06:13.604-07:00",
"status": "resolved",
"updated_at": "2026-10-22T20:16:42.373-07:00"
},
{
"components": [
{
"created_at": "2024-07-10T21:10:55.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 4",
"group_id": "x1089mr22815",
"id": "czhk96jtx93b",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.091-07:00"
},
{
"created_at": "2022-03-15T18:09:19.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 3",
"group_id": "z7txn2324j6k",
"id": "4fpjqbv7k6fp",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2021-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-12T12:31:45.889-07:00"
},
{
"created_at": "2024-09-12T17:05:44.740-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod Eu1",
"group_id": "6y3np02bqtxf",
"id": "y2wg26xx8r7r",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-12T12:31:45.914-07:00"
},
{
"created_at": "2024-11-07T20:37:07.688-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "rgdxklp8p0kd",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.261-07:00"
},
{
"created_at": "2021-06-01T14:48:02.147-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 1",
"group_id": "27qq03xf2msg",
"id": "kzjygtpl015t",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-04-10T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-12T12:31:45.964-07:00"
},
{
"created_at": "2021-06-01T14:56:05.591-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "hp77w1mwh2xg",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-04-10T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-12T12:31:45.990-07:00"
},
{
"created_at": "2024-11-07T20:34:14.643-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "pb161tg7ccrs",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.306-07:00"
},
{
"created_at": "2024-11-07T20:38:19.182-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "0kkbcr1h3cvx",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.350-07:00"
},
{
"created_at": "2024-11-07T20:40:44.997-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "trkh53c2bscy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.391-07:00"
},
{
"created_at": "2026-07-22T22:26:59.344-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 6",
"group_id": "1fpldmj5f61k",
"id": "8mjkl07z9kn1",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-12T12:31:46.093-07:00"
},
{
"created_at": "2024-11-07T20:32:37.209-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 1",
"group_id": "27qq03xf2msg",
"id": "n0m4jk53hrym",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.435-07:00"
},
{
"created_at": "2024-11-07T20:33:35.371-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "ctmk1q1dw7nr",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.484-07:00"
},
{
"created_at": "2024-11-07T20:40:12.699-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "th62w3qy4r3m",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.526-07:00"
},
{
"created_at": "2024-11-07T20:41:13.208-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "x3wy7hhj591c",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.579-07:00"
},
{
"created_at": "2026-07-22T22:27:49.688-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 6",
"group_id": "1fpldmj5f61k",
"id": "t8gybmf0b2qt",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.715-07:00"
},
{
"created_at": "2024-11-07T20:39:17.107-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod1",
"group_id": "27qq03xf2msg",
"id": "5znhlv1nwfcf",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.857-07:00"
},
{
"created_at": "2024-11-07T20:39:41.158-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod2",
"group_id": "qbxn0mh45jhv",
"id": "qvs27pfch8vy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.900-07:00"
},
{
"created_at": "2026-07-22T22:28:26.126-07:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod6",
"group_id": "1fpldmj5f61k",
"id": "q1369hjq7nn4",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.939-07:00"
}
],
"created_at": "2026-10-12T08:25:16.865-07:00",
"id": "th34d9m8qyc5",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "y2wg26xx8r7r",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "8mjkl07z9kn1",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-12T12:31:46.632-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-12T09:31:46.000-07:00",
"id": "drj45wj21b25",
"incident_id": "th34d9m8qyc5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T12:32:27.621-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "y2wg26xx8r7r",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "8mjkl07z9kn1",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently experiencing HTTP 429 (rate-limiting) errors when Harness Cloud builds access Maven Central, which may impact build execution. Our team is actively working with Sonatype to resolve the issue as quickly as possible. We will provide further updates as we make progress.",
"created_at": "2026-10-12T08:25:17.707-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-12T08:25:17.707-07:00",
"id": "z8syp51n6lv7",
"incident_id": "th34d9m8qyc5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T08:25:17.707-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Harness CI Cloud build issues when accessing Maven Central",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "tldr: \\(There was a brief rate limiting by maven central\\) \n\nMore details to follow",
"postmortem_body_last_updated_at": "2026-10-22T20:07:16.060-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-22T20:07:16.060-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-12T09:31:46.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/ffnvmypvjnqh",
"started_at": "2026-10-12T12:31:46.000-07:00",
"status": "resolved",
"updated_at": "2026-10-22T20:07:16.085-07:00"
},
{
"components": [
{
"created_at": "2024-04-17T02:36:03.213-07:00",
"description": "Harness Infrastructure as Code Management (IaCM) in Prod1",
"group_id": "27qq03xf2msg",
"id": "15bv7h1knvsk",
"name": "Infrastructure as Code Management (IaCM)",
"page_id": "zmnt6tkys0q0",
"position": 15,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-12T18:56:07.012-07:00"
}
],
"created_at": "2026-10-06T14:46:59.267-07:00",
"id": "2tcbk9tx6635",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-06T15:44:50.908-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T15:44:50.908-07:00",
"id": "djsn3v219s8r",
"incident_id": "2tcbk9tx6635",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T15:44:50.908-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "A fix has been implemented and we are monitoring the results. we have confirmation of this fixing the issue.",
"created_at": "2026-10-06T15:30:34.613-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T15:30:34.613-07:00",
"id": "6tb7ppjry25x",
"incident_id": "2tcbk9tx6635",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T15:30:34.613-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-06T15:30:11.483-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-06T15:30:11.483-07:00",
"id": "6l4csvkcnnbn",
"incident_id": "2tcbk9tx6635",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T15:30:11.483-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "major_outage"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-06T15:03:58.333-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-06T15:03:58.333-07:00",
"id": "pbcwpcj06x25",
"incident_id": "2tcbk9tx6635",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T15:03:58.333-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "major_outage",
"old_status": "partial_outage"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-10-06T14:47:57.203-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-06T14:47:57.203-07:00",
"id": "rz9wcwdt45kk",
"incident_id": "2tcbk9tx6635",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T14:47:57.203-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-06T14:46:59.375-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T14:46:59.375-07:00",
"id": "f1rjnb7vbbz5",
"incident_id": "2tcbk9tx6635",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T14:46:59.375-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-06T15:30:11.483-07:00",
"name": "Prod1: IACM Stages are failing and pipelines are intermittently failing to start.",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "# 1. Summary\n\nOn October 6, 2026, pipelines containing an IaCM stage were unable to start on Prod1 for approximately two hours. Affected users saw an error when the pipeline attempted to create its execution plan. Pipelines that did not use an IaCM stage continued to run normally throughout.\n\nThe underlying cause was an internal logging defect in the service that handles IaCM stage execution: under a specific, rare timing condition, that service's own logging system could end up waiting on itself indefinitely, which in turn stalled the broader process responsible for preparing pipeline execution plans. The issue was triggered by an unusually large burst of log activity from one customer's pipeline. We resolved the immediate impact by rolling the service back to its previous stable version, and a permanent code fix is currently in testing.\n\n# 2. Impact\n\n* All pipelines using an IaCM stage on Prod1 were unable to start for approximately two hours.\n* Multiple customers on the Prod1 cluster were affected.\n* Pipelines that did not use an IaCM stage were not affected and continued running normally.\n* No data was lost as a result of this incident.\n* As a side effect of the rollback used to resolve this incident, an unrelated fix for one customer \\(extending how long a certain authentication token remains valid during long-running pipelines\\) was temporarily reverted. That fix was reapplied the same evening.\n\n# 3. Root Cause\n\nThe service responsible for IaCM pipeline stages logs its activity through an internal queue: log messages are placed on the queue, and a single background worker continuously reads from it and writes the messages out. Separately, that service has a filter that can rewrite the severity of certain log messages back onto that same queue.\n\nUnder a specific, narrow timing condition, this created a deadlock, if the queue happened to be completely full at the exact moment a rewritten message was being submitted back onto it, the one worker responsible for freeing up space on the queue would itself get stuck waiting for space to free up, a condition it could never resolve, since it was the only thing that could have freed that space. Once that happened, every other part of the service that needed to log anything also became stuck waiting, which cascaded into the pipeline-plan-creation process failing for any pipeline using an IaCM stage.\n\n# 4. Resolution\n\n* Rolled the service back to its last stable version, which resolved the immediate incident.\n\n# 5. Preventive Actions\n\n1. **Optimize the underlying defect**: Optimize and tune the logging filter so that rewriting a message's severity happens on the thread that generated the log line, rather than being resubmitted through the same background worker's queue, eliminating the deadlock condition entirely.\n2. **Enhance monitoring on logging-queue health**: add metrics and alerting on this internal queue so that this class of saturation pages our on-call team automatically rather than relying on customer reports.\n3. **Evaluate limits and perform high stress tests with very large pipelines**:",
"postmortem_body_last_updated_at": "2026-10-22T20:05:26.000-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-22T20:05:26.000-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-06T15:44:50.908-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/97z6j192pz3j",
"started_at": "2026-10-06T14:46:59.256-07:00",
"status": "resolved",
"updated_at": "2026-10-22T20:05:26.025-07:00"
},
{
"components": [],
"created_at": "2026-10-06T13:00:00.000-07:00",
"id": "2366tsdd6fwj",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": null,
"body": "On October 7, 2026, from 08:00 UTC to 11:00 UTC (1:30 PM to 4:30 PM IST), some pipeline executions on the Prod 7 cluster saw intermittent failures in Policy (OPA) step evaluations.",
"created_at": "2026-10-16T03:41:36.009-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-06T13:00:00.000-07:00",
"id": "cfccd1mm36r8",
"incident_id": "2366tsdd6fwj",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-16T03:41:36.009-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Intermittent failures in Policy step evaluations on Prod 7",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-06T13:00:00.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/pfwpb7pvmlf7",
"started_at": "2026-10-16T03:41:35.945-07:00",
"status": "resolved",
"updated_at": "2026-10-16T03:41:45.771-07:00"
},
{
"components": [
{
"created_at": "2024-04-17T02:36:03.213-07:00",
"description": "Harness Infrastructure as Code Management (IaCM) in Prod1",
"group_id": "27qq03xf2msg",
"id": "15bv7h1knvsk",
"name": "Infrastructure as Code Management (IaCM)",
"page_id": "zmnt6tkys0q0",
"position": 15,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-12T18:56:07.012-07:00"
}
],
"created_at": "2026-10-06T04:58:34.546-07:00",
"id": "k22j0ptxcq10",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-06T09:21:00.773-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T09:21:00.773-07:00",
"id": "93fdg8lw5k13",
"incident_id": "k22j0ptxcq10",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T09:21:00.773-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-06T05:24:36.863-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T05:24:36.863-07:00",
"id": "l6lkky90m6c7",
"incident_id": "k22j0ptxcq10",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T05:24:36.863-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-10-06T04:59:39.013-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T04:59:39.013-07:00",
"id": "lkfwwtdpx4c6",
"incident_id": "k22j0ptxcq10",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T04:59:39.013-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-06T04:58:34.666-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T04:58:34.666-07:00",
"id": "d94sv29f3vm9",
"incident_id": "k22j0ptxcq10",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T04:58:34.666-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Single Customer: IaCM pipeline executions failing intermittently in Prod 1",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "## Summary\n\nOn **Tuesday, 6 October 2026**, IaCM pipelines that use the **OpenTofu plugin** in **Prod1** failed intermittently for a production accountAffected runs injected secrets successfully, then later steps timed out or failed authentication. The same pipeline often succeeded on retry, with no configuration change. Typical failing executions ran for about 90 minutes.\n\nOther Harness modules were not broadly impacted. We have no report of customer data loss. Pipelines that finished inside the token lifetime were unaffected by this defect.\n\n## Root Cause\n\nThe IaCM stages minted authentication tokens that expired after one hour. Policy checks later in a long run then rejected the token, so the step timed out or failed authentication.\n\nThat one-hour lifetime was fixed. Runs of about 90 minutes crossed the one-hour mark, which is why the failures showed up on longer pipelines and often cleared on a shorter retry.\n\nA separate set of plan errors in the same account came from credential setup in the customer workspace and is outside this incident.\n\n## Impact\n\n* **Product area:** IaCM pipelines using the IaCM plugins talking to OPA sdk.\n* **Customer experience:** intermittent timeouts and authentication failures after secret injection, on runs longer than about one hour.\n* **Environment:** Prod1, one production account \n* **Data:** No data loss reported.\n\n\n\n## Remediation\n\n* **Immediate:** Opened a status page for the affected Prod1 account, and shipped an interim IaCM manager change for one class of the observed failures while the plugin fix was built.\n* **Permanent:** Shipped an OpenTofu plugin hotfix so token expiry is configurable. In production the lifetime was raised to about 6 hours, so a long pipeline does not outlive the token. The hotfix was validated in QA and deployed to Prod1.\n\nWe apologise for the disruption caused. Please reach out to your Harness support contact if you have any further questions.",
"postmortem_body_last_updated_at": "2026-10-09T12:16:28.803-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-09T12:16:28.803-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-06T09:21:00.773-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/83x3q34g66rs",
"started_at": "2026-10-06T04:58:34.535-07:00",
"status": "resolved",
"updated_at": "2026-10-09T12:16:28.830-07:00"
},
{
"components": [
{
"created_at": "2022-03-15T18:10:49.160-07:00",
"description": "Harness Continuous Delivery Next Generation (CDNG) in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "gbltvmpdkb3r",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": true,
"start_date": "2021-10-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-09-29T08:54:15.259-07:00"
}
],
"created_at": "2026-10-06T05:54:35.418-07:00",
"id": "lrt7f345h0vj",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "gbltvmpdkb3r",
"name": "Prod 2 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-06T08:35:29.440-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T08:35:29.440-07:00",
"id": "h2m9q7zlq6v2",
"incident_id": "lrt7f345h0vj",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T08:35:29.440-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "gbltvmpdkb3r",
"name": "Prod 2 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-06T05:54:35.493-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T05:54:35.493-07:00",
"id": "rk06v1h514f8",
"incident_id": "lrt7f345h0vj",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T05:54:35.493-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-06T05:54:35.493-07:00",
"name": "Prod2 Pipelines with verify step failing intermittently",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "### Summary\n\nCustomers on Prod 2 experienced intermittent failures of a few Verify steps at startup. Affected steps failed shortly after launch with a message indicating Continuous Verification\\(CV\\) could not reach an internal Harness service. The issue stemmed from a recent service update that introduced connection pooling to improve performance and reliability with a keep-alive setting incompatible with the receiving service's timeout.\n\nNo customer data was lost or corrupted.\n\n### Customer Impact\n\nCustomers using CV Verify steps on Prod-2 saw intermittent step failures during the incident window. The failure rate was low — the majority of executions completed successfully throughout the window. Pipelines that did not use CV Verify steps were unaffected.\n\n### Root Cause\n\nA recent CV service update introduced connection pooling to an internal platform component, with a keep-alive duration longer than that component's idle timeout. Connections idle long enough were silently closed by the server; the CV service did not detect this and had no retry logic for such failures, causing them to surface directly as Verify step failures. The issue was intermittent because it only affected connections that happened to be idle long enough to expire.\n\n### Mitigation\n\nThe connection keep-alive duration was reduced to fall within the receiving service's idle timeout. Failures dropped to near zero immediately, and no further step failures of this type have been observed on any production environment.\n\n### Next Steps\n\nTo prevent recurrence, Harness will:\n\n* **Fix connection handling:** Tuned the default keep-alive and enabled automatic retry on connection failures so stale-connection errors are handled transparently.\n* **Add resilience at step startup:** Introduce retry with backoff for the internal service call at the Verify step start, so isolated transient failures do not surface as immediate step failures.\n* **Improve alerting:** Enhanced monitoring for internal connection error rates.",
"postmortem_body_last_updated_at": "2026-10-09T18:57:18.023-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-09T18:57:18.023-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-06T08:35:29.440-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/31bpzf9tw2c0",
"started_at": "2026-10-06T05:54:35.408-07:00",
"status": "resolved",
"updated_at": "2026-10-09T18:57:18.050-07:00"
},
{
"components": [
{
"created_at": "2025-08-27T10:29:44.922-07:00",
"description": "Core platform constructs eg. Platform delegate (eg. Delegate communication), Platform Core (eg. Account, Login, Project/Org, Connectors, Secrets, Delegate, Settings, Notifications, Audits, Templates, Services, Environments , Policies, File Store, Log Uploads), Platform Authentication \u0026 Authorization (login, RBAC, logout, API keys, Service account, Tokens etc.)",
"group_id": "x1089mr22815",
"id": "561g9b7sqgct",
"name": "Platform",
"page_id": "zmnt6tkys0q0",
"position": 17,
"showcase": true,
"start_date": "2025-08-27T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.212-07:00"
},
{
"created_at": "2025-08-27T10:31:10.310-07:00",
"description": "Core platform constructs eg. Platform delegate (eg. Delegate communication), Platform Core (eg. Account, Login, Project/Org, Connectors, Secrets, Delegate, Settings, Notifications, Audits, Templates, Services, Environments , Policies, File Store, Log Uploads), Platform Authentication \u0026 Authorization (login, RBAC, logout, API keys, Service account, Tokens etc.)",
"group_id": "6y3np02bqtxf",
"id": "zp7rk1rtt25b",
"name": "Platform",
"page_id": "zmnt6tkys0q0",
"position": 17,
"showcase": true,
"start_date": "2025-08-27T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-05T17:26:43.191-07:00"
},
{
"created_at": "2025-08-27T10:30:07.277-07:00",
"description": "Core platform constructs eg. Platform delegate (eg. Delegate communication), Platform Core (eg. Account, Login, Project/Org, Connectors, Secrets, Delegate, Settings, Notifications, Audits, Templates, Services, Environments , Policies, File Store, Log Uploads), Platform Authentication \u0026 Authorization (login, RBAC, logout, API keys, Service account, Tokens etc.)",
"group_id": "27qq03xf2msg",
"id": "vy68z1vstc62",
"name": "Platform",
"page_id": "zmnt6tkys0q0",
"position": 20,
"showcase": true,
"start_date": "2025-08-27T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-05T17:26:43.213-07:00"
},
{
"created_at": "2025-08-27T10:30:21.767-07:00",
"description": "Core platform constructs eg. Platform delegate (eg. Delegate communication), Platform Core (eg. Account, Login, Project/Org, Connectors, Secrets, Delegate, Settings, Notifications, Audits, Templates, Services, Environments , Policies, File Store, Log Uploads), Platform Authentication \u0026 Authorization (login, RBAC, logout, API keys, Service account, Tokens etc.)",
"group_id": "z7txn2324j6k",
"id": "g5xm7jlw5rr4",
"name": "Platform",
"page_id": "zmnt6tkys0q0",
"position": 20,
"showcase": true,
"start_date": "2025-08-27T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-05T17:26:43.234-07:00"
},
{
"created_at": "2025-08-27T10:30:51.938-07:00",
"description": "Core platform constructs eg. Platform delegate (eg. Delegate communication), Platform Core (eg. Account, Login, Project/Org, Connectors, Secrets, Delegate, Settings, Notifications, Audits, Templates, Services, Environments , Policies, File Store, Log Uploads), Platform Authentication \u0026 Authorization (login, RBAC, logout, API keys, Service account, Tokens etc.)",
"group_id": "qbxn0mh45jhv",
"id": "0yrdl70cplw9",
"name": "Platform",
"page_id": "zmnt6tkys0q0",
"position": 20,
"showcase": true,
"start_date": "2025-08-27T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-05T17:26:43.255-07:00"
}
],
"created_at": "2026-10-05T15:01:20.783-07:00",
"id": "x4lb8nxlx8gp",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "vy68z1vstc62",
"name": "Prod 1 - Platform",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "0yrdl70cplw9",
"name": "Prod 2 - Platform",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "g5xm7jlw5rr4",
"name": "Prod 3 - Platform",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "zp7rk1rtt25b",
"name": "Prod Eu 1 - Platform",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Verified the fix and resolved SSO issues.",
"created_at": "2026-10-05T17:39:05.826-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-05T17:39:05.826-07:00",
"id": "vk9hdqn801j7",
"incident_id": "x4lb8nxlx8gp",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-05T17:39:05.826-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "vy68z1vstc62",
"name": "Prod 1 - Platform",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "0yrdl70cplw9",
"name": "Prod 2 - Platform",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "g5xm7jlw5rr4",
"name": "Prod 3 - Platform",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "zp7rk1rtt25b",
"name": "Prod Eu 1 - Platform",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "We are continuing to monitor for any further issues.",
"created_at": "2026-10-05T17:26:43.290-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-05T17:26:43.290-07:00",
"id": "8zqtj2blx3qn",
"incident_id": "x4lb8nxlx8gp",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-05T17:26:43.290-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "vy68z1vstc62",
"name": "Prod 1 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "0yrdl70cplw9",
"name": "Prod 2 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "g5xm7jlw5rr4",
"name": "Prod 3 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "zp7rk1rtt25b",
"name": "Prod Eu 1 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-05T17:26:22.378-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-05T17:26:22.378-07:00",
"id": "0ljyf47htcq2",
"incident_id": "x4lb8nxlx8gp",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-05T17:26:22.378-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "vy68z1vstc62",
"name": "Prod 1 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "0yrdl70cplw9",
"name": "Prod 2 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "g5xm7jlw5rr4",
"name": "Prod 3 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "zp7rk1rtt25b",
"name": "Prod Eu 1 - Platform",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-05T16:50:45.541-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-05T16:50:45.541-07:00",
"id": "pzj7b9fv1n7s",
"incident_id": "x4lb8nxlx8gp",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-05T16:50:45.541-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "vy68z1vstc62",
"name": "Prod 1 - Platform",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "0yrdl70cplw9",
"name": "Prod 2 - Platform",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "g5xm7jlw5rr4",
"name": "Prod 3 - Platform",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "zp7rk1rtt25b",
"name": "Prod Eu 1 - Platform",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-05T15:01:21.058-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-05T15:01:21.000-07:00",
"id": "n8ttnhj7z26q",
"incident_id": "x4lb8nxlx8gp",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-05T15:02:48.324-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-05T17:26:22.378-07:00",
"name": "Hosted MCP Logins W/ SSO Failing",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "## Okta Application SP-Initiated SSO Login Failures\n\n\n\n# Summary\n\n\n\nBetween October 5, 2026, 18:18 UTC and October 6, 2026, 00:26 UTC, some Harness customers were unable to sign in to the Harness Hosted MCP server via SAML Single Sign-On \\(SSO\\). The main Harness Portal user authentication remained fully operational. Affected users saw login errors or were redirected to an invalid SSO page instead of completing sign-in.\n\n\n\nThe issue was caused by breaking changes in a version upgrade of the Harness identity service HarnessID. We fixed it with a series of hotfixes, and SSO login was fully restored for all affected customers.\n\n\n\n# Impact\n\n\n\nUsers signing in to Harness MCP via SAML SSO were unable to complete authentication. Existing MCP sessions that relied on SSO could not be refreshed, so some users had to sign in again and could not. Non-SSO logins to Hosted MCP worked during most of the incident. No customer data was lost or exposed.\n\nCustomers using their own generic SAML 2.0 application configuration in Okta were not affected.\n\n# Root Cause\n\nOn September 29, 2026, an updated version of the Harness application was published to the Okta Integration Network. The update added support for registering multiple reply \\(ACS\\) URLs, so that customers running Harness across several regions or environments could configure more than one callback URL.\n\n\n\nHarness includes an AssertionConsumerServiceURL value in the SAML authentication request it sends to Okta. Before this update, Okta did not validate that value for application instances without additional ACS URLs configured and instead returned the SAML assertion to the application's default ACS URL. After the update, this validation became active across all existing application instances, including those where the customer had never configured an additional ACS URL. Because the requested value was not present in the instance's registered list, Okta returned an HTTP 400 error.\n\n\n\nThe updated application went through Okta's standard OIN publishing review, which includes backward-compatibility verification against existing installations of the published integration. This change in validation behavior for instances with no additional ACS URLs configured was not surfaced during that review, so it reached customers at the time of release.\n\n# Remediation\n\n**Immediate Response**\n\n\n\nTwo workarounds were published on the status page: sign in from the Harness tile in Okta \\(IdP-initiated login\\), which was unaffected, or register the application's Okta SSO URL in the Additional ACS URLs field of the Harness Okta application. Harness engaged each reported affected customer directly and validated their sign-in.\n\n\n\n**Resolution**\n\n\n\nRolling the application back to the previous version was deliberately avoided, as that would only defer the same disruption to the next release. Every affected customer was instead unblocked through the ACS URL registration, and we are working with Okta to make the ACS URL validation conditional again, so that instances with no additional ACS URLs configured behave exactly as they did before, and no customer action is required.\n\n# Action Items\n\nTo prevent such issues from happening again, Harness will take the following action items\n\n\n\n1. Working with Okta to restore conditional ACS URL validation, so instances with no additional ACS URLs configured behave exactly as before and no customer action is required.\n2. Publish a dedicated configuration guide for the Harness OIN application in Harness documentation, including reply \\(ACS\\) URL requirements.\n3. Add an independent pre-release verification step against existing customer configurations for any future update to the published Okta application, alongside Okta's own review.\n4. Migrate user authentication to Harness ID, which removes the dependency on this validation behavior entirely.",
"postmortem_body_last_updated_at": "2026-10-09T09:38:09.840-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-09T09:17:30.829-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-05T17:39:05.826-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/vw0np3lrbj89",
"started_at": "2026-10-05T15:01:20.756-07:00",
"status": "resolved",
"updated_at": "2026-10-09T09:38:09.843-07:00"
}
],
"page": {
"id": "zmnt6tkys0q0",
"name": "Harness",
"url": "https://status.harness.io"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}