Snapshot 253071
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"nxbqtxqrm55f",
"xs5vhwly9mk5",
"f347vm5yn95f",
"vxl2rz20dcyr",
"yyv8hjtmvpzv",
"cc654c3j6476",
"kzjygtpl015t",
"n0m4jk53hrym",
"5znhlv1nwfcf",
"83n5xv3hpb69",
"by80qyspb2jq",
"sn84zlkwcg24",
"lv49p6nbl193",
"5qd852klgxc0",
"15bv7h1knvsk",
"mn3934kz3w69",
"5rys3t6403zn",
"1lccfpfwmh4y",
"rz6k0bcnrpr1",
"vy68z1vstc62",
"l539c2tfydzj",
"8vf2zpmhfyy0",
"j4tq5500kmg8",
"77bh7sgmbbv3"
],
"created_at": "2021-06-01T14:46:03.030-07:00",
"description": null,
"group_id": null,
"id": "27qq03xf2msg",
"name": "Prod 1",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.090-07:00"
},
{
"components": [
"s54kkcr6t0gq",
"gbltvmpdkb3r",
"bqhhx747whp3",
"tgcylzz5nlry",
"c6myzvwng2lw",
"9677ldz5y7sd",
"hp77w1mwh2xg",
"ctmk1q1dw7nr",
"qvs27pfch8vy",
"65kdr9mrlblc",
"cg2xjl1jzk0m",
"qbzy2sc4skqh",
"gdlfxhr31qmb",
"sbwzykrf1q8t",
"b4tvy5s9wr6z",
"v1slhlqmmbm1",
"5k3yywp3xvqj",
"640z11cmjv3n",
"fwh6z4glx958",
"0yrdl70cplw9",
"czfjkrnqlcsk",
"9pr1l3rwf14n",
"8tnc91hlt8d2"
],
"created_at": "2021-06-01T14:53:31.712-07:00",
"description": null,
"group_id": null,
"id": "qbxn0mh45jhv",
"name": "Prod 2",
"page_id": "zmnt6tkys0q0",
"position": 3,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.103-07:00"
},
{
"components": [
"rwx7y5zv4yq5",
"l1sx8bck1yzb",
"fw0zmk94bkpc",
"tq8rdbq9np4d",
"brmlw2942fbc",
"4fpjqbv7k6fp",
"pb161tg7ccrs",
"th62w3qy4r3m",
"kxxh18chwqlg",
"k4sqhp5t769d",
"nw83hg1v0ftp",
"q489v85zw6qs",
"vf6qxw4l52m4",
"n0f6mg62hgx3",
"ths5wq122vtx",
"804lzkqf2vfk",
"jjmrjld2vwnb",
"yhy2w4n4q7kk",
"sbm4gmjj60nl",
"g5xm7jlw5rr4",
"ztrdyp8d48sx",
"mbpd7mcgc0l0"
],
"created_at": "2021-06-01T14:54:00.058-07:00",
"description": null,
"group_id": null,
"id": "z7txn2324j6k",
"name": "Prod 3",
"page_id": "zmnt6tkys0q0",
"position": 4,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.115-07:00"
},
{
"components": [
"p9t9g01vgy47",
"5161jzbx0d8w",
"j9qkdrxdylw2",
"bp1hsfllmds0",
"czhk96jtx93b",
"rgdxklp8p0kd",
"trkh53c2bscy",
"sfc57k1p0sqp",
"790nrgxflg5x",
"k0hbqz5c81v8",
"h1wfypqglf49",
"gvhsp1j79nbm",
"xzg7lrrqyk37",
"nxsthnf9gk3g",
"6xhczbg7xfyv",
"0vlv5pjpjzbs",
"561g9b7sqgct"
],
"created_at": "2024-07-10T21:08:30.585-07:00",
"description": null,
"group_id": null,
"id": "x1089mr22815",
"name": "Prod 4",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2024-10-28T19:09:00.126-07:00"
},
{
"components": [
"zbyg84df96x7",
"007h1ztzxtlw",
"v7mz6ltk413q",
"w3csxhc0mqz7",
"rbn7hjb2qf8r",
"y2wg26xx8r7r",
"0kkbcr1h3cvx",
"x3wy7hhj591c",
"z9267hzwpnwq",
"fwvncn7t50n9",
"r87dg7ccrjpv",
"lg2vtjgxqkzw",
"y3rkhygkv1g3",
"1b70wzgp8spj",
"3hdhfjy41l7p",
"rh8y3gywr698",
"zp7rk1rtt25b",
"yhy0vm97y09n"
],
"created_at": "2024-09-09T21:49:39.223-07:00",
"description": null,
"group_id": null,
"id": "6y3np02bqtxf",
"name": "Prod Eu 1",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-22T04:49:45.778-07:00"
},
{
"created_at": "2021-12-10T16:29:47.633-08:00",
"description": "Harness OverOps",
"group_id": null,
"id": "1h5qbtxd2622",
"name": "Service Reliability Management - Error Tracking FirstGen (fka OverOps)",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-12-10T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2024-10-28T19:09:00.147-07:00"
},
{
"components": [
"52zt8q9l5yk4",
"wjt25yrgfl1d"
],
"created_at": "2025-01-13T08:34:11.290-08:00",
"description": null,
"group_id": null,
"id": "rx8vt1qyrpsv",
"name": "Software Engineering Insights FirstGen (fka Propelo)",
"page_id": "zmnt6tkys0q0",
"position": 10,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-01-13T08:34:11.290-08:00"
},
{
"components": [
"hnbf7sq0cq5b",
"s0dh2g03zz7r",
"fbbt3z4ytn23",
"yxvnmfxvggb9",
"0r0gmbxty2d9",
"dnkfpsn0r5vr"
],
"created_at": "2025-09-01T23:44:00.402-07:00",
"description": "Traceable Clusters",
"group_id": null,
"id": "np782snlmj00",
"name": "Traceable",
"page_id": "zmnt6tkys0q0",
"position": 11,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2025-09-01T23:44:29.556-07:00"
},
{
"components": [
"qzlxgjbrdby4",
"gflflybxx6b8",
"zwf5f0yg33qk",
"w90818668zwc",
"xwbxwtcl9g25",
"y21hwc1hhpnz",
"qcj53m9f1ptq",
"6lmbj16bvn16"
],
"created_at": "2025-11-05T10:12:41.367-08:00",
"description": "Feature Management \u0026 Experimentation",
"group_id": null,
"id": "wm26z10lksx3",
"name": "Feature Management \u0026 Experimentation (FME)",
"page_id": "zmnt6tkys0q0",
"position": 13,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-02-04T14:36:46.108-08:00"
},
{
"components": [
"s1dlyyndwjl6",
"6k06w76wqwsc",
"p20hnvtnpw1b",
"drq1sbn8hw11",
"v5rf21nbqwg2"
],
"created_at": "2026-01-06T02:08:28.178-08:00",
"description": "Dedicated SAAS cluster for Goldman Sachs",
"group_id": null,
"id": "2w5qsvz2lk4n",
"name": "Prod 7",
"page_id": "zmnt6tkys0q0",
"position": 14,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-01-06T02:11:59.702-08:00"
},
{
"components": [
"0rxrdlk4dr3m",
"snxfjcdrnddd",
"lfrwy26bjy27",
"s9f6pwt8tbj1",
"gl4wwd0s6l4n",
"7jfnyljgkhsc"
],
"created_at": "2026-01-06T02:35:45.999-08:00",
"description": "Dedicated SAAS cluster for Barclays",
"group_id": null,
"id": "n6rk38cphg56",
"name": "Prod 8",
"page_id": "zmnt6tkys0q0",
"position": 15,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-01-06T02:36:31.092-08:00"
},
{
"components": [
"shg48w7t35l8",
"1zmt4tlpxh7d",
"tsvcht7ggxpq",
"6q2y478qnshb",
"5vjk880lv3yb",
"vqy2pfsw7sxx",
"gfsntx0t1n45",
"glxwgv0pdjrb",
"6cq3s06svg8n",
"d7b0scq7bvf5",
"xckv4nnb4hjy",
"t9pvbwy0smfp",
"m25krg585c3z",
"7fp2gfxytcw0",
"l5np7vy2h0bc",
"kbjf89kfnxfm",
"z1r28087zxf5",
"gm8yq9pr4xbk",
"47jx56p992v7",
"j4mnhf18ky22",
"93j99d3r0nb3",
"mttlsgnqy0mw",
"1yjpjqbtb1xn"
],
"created_at": "2026-04-14T01:59:10.412-07:00",
"description": null,
"group_id": null,
"id": "j9f15dxpqydw",
"name": "Dev",
"page_id": "zmnt6tkys0q0",
"position": 16,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-09-11T07:30:50.641-07:00"
},
{
"components": [
"tgbvl8jm69gd",
"n48l3qtwftyq",
"9nh0xfkz9hkq",
"jddlj6k9xs29"
],
"created_at": "2026-04-20T02:54:03.635-07:00",
"description": null,
"group_id": null,
"id": "23r8514jn030",
"name": "Prod 5",
"page_id": "zmnt6tkys0q0",
"position": 17,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-04-20T02:55:08.885-07:00"
},
{
"created_at": "2026-06-19T02:52:03.505-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Dev",
"group_id": null,
"id": "9r4nnf9jzl4t",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-06-19T02:52:03.505-07:00"
},
{
"components": [
"y41n6v53x5z1",
"hbt1g5bqvg1l",
"hjr9dqb9t38t",
"th0ygdtzlvdk",
"t45ncxygrpxm",
"nyhhbjtgtst4",
"8mjkl07z9kn1",
"t8gybmf0b2qt",
"q1369hjq7nn4",
"60cy4p5vr6gp",
"xbqdy20jrrs5",
"skccvt241051",
"52ny1ymph3wd",
"2cn1374658lx",
"r09j2jvg6r3h",
"l0fd5d66jnpp",
"kryxb6g45sqn",
"lbdf8v2kkq2j",
"hg36cdlm95bl",
"fhb7jrwcfdzl",
"w9kcl4xznsym",
"jdc2stt9jvds",
"kf7xlcfh77bx"
],
"created_at": "2026-07-22T22:21:45.295-07:00",
"description": null,
"group_id": null,
"id": "1fpldmj5f61k",
"name": "Prod 6",
"page_id": "zmnt6tkys0q0",
"position": 19,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-22T22:21:45.295-07:00"
}
],
"incidents": [
{
"components": [],
"created_at": "2026-11-05T16:59:06.522-08:00",
"id": "14wd7nkgxlh4",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "This incident has been resolved.",
"created_at": "2026-11-05T23:13:30.657-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-05T23:13:30.657-08:00",
"id": "8qxphg54gjxp",
"incident_id": "14wd7nkgxlh4",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-05T23:13:30.657-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-11-05T17:16:16.936-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-05T17:16:16.936-08:00",
"id": "779zsv4s2ks9",
"incident_id": "14wd7nkgxlh4",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-05T17:16:16.936-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We are currently investigating this issue.",
"created_at": "2026-11-05T16:59:06.573-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-05T16:59:06.573-08:00",
"id": "3ws1c8ssdn5b",
"incident_id": "14wd7nkgxlh4",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-05T16:59:06.573-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "apidocs.harness.io not responding to HTTPS requests",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-05T23:13:30.657-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/kftdhhdfp3pq",
"started_at": "2026-11-05T16:59:06.516-08:00",
"status": "resolved",
"updated_at": "2026-11-05T23:13:30.679-08:00"
},
{
"components": [],
"created_at": "2026-11-02T17:01:23.020-08:00",
"id": "96kl099bnr4f",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "The provider has resolved the issue and service is restored.",
"created_at": "2026-11-03T18:33:11.735-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T18:33:11.000-08:00",
"id": "p1nz748w0q59",
"incident_id": "96kl099bnr4f",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T18:33:51.618-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-11-03T04:41:37.652-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T04:41:37.652-08:00",
"id": "vbxgbm6p8rjq",
"incident_id": "96kl099bnr4f",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T04:41:37.652-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "The provider is currently experiencing an outage unrelated to Harness infrastructure and is investigating it. We are actively monitoring the provider’s status and will provide an update when the service is restored.",
"created_at": "2026-11-03T03:09:16.895-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-03T03:09:16.000-08:00",
"id": "jpp1c7ydfj6t",
"incident_id": "96kl099bnr4f",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-03T03:21:25.221-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-11-02T19:48:56.566-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-02T19:48:56.566-08:00",
"id": "hvnh0tgx97cf",
"incident_id": "96kl099bnr4f",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-02T19:48:56.566-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "developer.harness.io is currently degraded or unavailable due to an issue with our external hosting provider. This is outside Harness infrastructure. We're monitoring the provider's status and will update when service is restored.",
"created_at": "2026-11-02T17:02:40.852-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-02T17:02:40.852-08:00",
"id": "9p50pmjby7fm",
"incident_id": "96kl099bnr4f",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-02T17:02:40.852-08:00",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "We are currently investigating this issue.",
"created_at": "2026-11-02T17:01:23.075-08:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-11-02T17:01:23.075-08:00",
"id": "hjrd36k18ltm",
"incident_id": "96kl099bnr4f",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-11-02T17:01:23.075-08:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-11-02T19:48:56.566-08:00",
"name": "developer.harness.io not responding to HTTPS requests",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-11-03T18:33:11.000-08:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/y8g2t5wrx3j1",
"started_at": "2026-11-03T03:09:16.000-08:00",
"status": "resolved",
"updated_at": "2026-11-03T18:33:51.640-08:00"
},
{
"components": [
{
"created_at": "2024-07-10T21:10:55.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 4",
"group_id": "x1089mr22815",
"id": "czhk96jtx93b",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.784-07:00"
},
{
"created_at": "2022-03-15T18:09:19.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 3",
"group_id": "z7txn2324j6k",
"id": "4fpjqbv7k6fp",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2021-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.833-07:00"
},
{
"created_at": "2021-06-01T14:48:02.147-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 1",
"group_id": "27qq03xf2msg",
"id": "kzjygtpl015t",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-04-10T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.883-07:00"
},
{
"created_at": "2021-06-01T14:56:05.591-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "hp77w1mwh2xg",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-04-10T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.931-07:00"
}
],
"created_at": "2026-10-31T20:43:11.430-07:00",
"id": "bmyd74x3h4jy",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-10-31T22:34:51.838-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T22:34:51.838-07:00",
"id": "xr4f3c1x9fnp",
"incident_id": "bmyd74x3h4jy",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T22:34:51.838-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "Verification is currently underway for the maintenance items.",
"created_at": "2026-10-31T22:27:44.998-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T22:27:44.998-07:00",
"id": "s7r3prb66wx0",
"incident_id": "bmyd74x3h4jy",
"status": "verifying",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T22:27:44.998-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-10-31T21:47:55.852-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T21:47:55.852-07:00",
"id": "2fcwxnkqjs47",
"incident_id": "bmyd74x3h4jy",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T21:47:55.852-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Hello,\n\nWe will perform scheduled maintenance on Hosted Mac Builds to address eMudhra Certificate Authority (CA) distrust updates.\n\nImpact: Hosted Mac Builds will be unavailable during the maintenance window across all production clusters.\n\nPlease plan your builds accordingly. If you have questions or need assistance validating your pipelines, contact Harness Support.",
"created_at": "2026-10-31T20:43:11.524-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-31T20:43:11.524-07:00",
"id": "mg1f4pjzgt1h",
"incident_id": "bmyd74x3h4jy",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-31T20:43:11.524-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-31T22:27:44.998-07:00",
"name": "Hosted Mac Builds",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-31T22:34:51.838-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": "2026-10-31T21:30:00.000-07:00",
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-31T23:30:00.000-07:00",
"shortlink": "https://stspg.io/sk86ygzg3fng",
"started_at": "2026-10-31T20:43:11.408-07:00",
"status": "completed",
"updated_at": "2026-10-31T22:34:51.884-07:00"
},
{
"components": [],
"created_at": "2026-10-28T10:10:38.000-07:00",
"id": "7kbnml9dyfd0",
"impact": "none",
"impact_override": null,
"incident_updates": [
{
"affected_components": null,
"body": "Starting 10/28/2026 10:10 AM CI jobs on internal VM runners intermittently hung or expired without logs, affecting some customers in prod1/2. The resolution is to upgrade the new runner version https://github.com/drone-runners/drone-runner-aws/releases/tag/v1.0.0-rc.318 (Docker image: drone/drone-runner-aws:1.0.0-rc.318)",
"created_at": "2026-10-28T12:22:38.265-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-28T10:10:38.000-07:00",
"id": "tk6kd6t940cv",
"incident_id": "7kbnml9dyfd0",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-28T12:23:42.184-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Intermittent CI Pipeline using VM Runner slowness and errors",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "# 1. Summary\n\nFrom October 28, 2026, 10:14 AM PT to October 29, 2026, 9:12 PM PT, CI pipelines using VM-based runners intermittently hung or expired after initialization, often with no step logs or console output. The issue affected two Prod1 customers, with additional VM-runner customers identified for proactive follow-up across Prod1, Prod2, and Prod3. The trigger was message-queue backpressure caused by broker slowness following an infrastructure image upgrade in that layer; we mitigated by rolling back that change, which resolved the platform-side trigger within about two hours. Follow-up found the two affected customers were running older versions of our VM runner and its companion log-streaming component that predated a fix for a related deadlock issue, so the customer-facing resolution was to pin to a specific, validated runner version instead of relying on a floating \"latest\" tag.\n\n# 2. Impact\n\n* CI jobs on VM runners could hang mid-run, time out after long waits , or expire without usable logs.\n* This blocked deployments for affected pipelines and slowed diagnosis, since failures were often silent\n* Two Prod1 customers were confirmed affected; a broader set of VM-runner customers across Prod1, Prod2, and Prod3 were identified as potentially exposed and proactively contacted.\n* Impact was limited to CI workflows using VM-based runners; other execution types were not affected.\n\n# 3. Root Cause\n\nOur CI VM-runner architecture streams step logs through a live-logging pipeline backed by a message-queue layer. During this incident, that queue experienced backpressure, following an infrastructure image upgrade on one of the queue's brokers. This degraded the live-log streaming path platform-wide, causing some in-flight log writes to stall.\n\n\n\nHowever, this platform-level degradation was transient and self-healed once we rolled back the image upgrade. It only turned into multi-hour, customer-visible hangs for customers who were running older builds of our VM runner and its companion log-streaming helper component,specifically, builds that predated a fix for a known deadlock condition in how that component handled interrupted log streams. Under normal queue conditions, this defect was dormant; the queue backpressure was enough to trigger it on those older builds, causing their executions to hang on an internal request until a roughly 4-hour timeout expired.\n\nBoth confirmed affected customers were using runner versions older than the version containing the fix, and in one case the customer believed they were on the latest available image but were, in practice, still running an older pulled version — because our \"latest\" tag at the time did not reliably correspond to the newest, QA-validated build.\n\n# 4. Resolution\n\n* Identified that failures were specific to VM-runner executions hanging after initialization, and separated this from an unrelated initialization-stage incident from the prior day.\n* Traced message-queue producer errors and broker slowness to a recent infrastructure image upgrade on that queue layer.\n* Rolled back that image upgrade, which brought queue/log-streaming error rates to zero within about two hours of incident start.\n* Confirmed, with the two affected customers, the specific runner and log-streaming component versions they were running.\n* Recommended and confirmed a specific, pinned runner version containing the deadlock fix as the customer-facing resolution.\n* Identified additional VM-runner customers for proactive outreach and validation across Prod1, Prod2, and Prod3.\n\n# 5. Preventive Actions\n\n1. **Surface the VM runner version** directly in pipeline execution logs and metadata, so this no longer has to be manually confirmed with customers during an incident.\n2. **Publish guidance** recommending customers pin to an explicit runner version rather than relying on \"latest\".\n3. **Add detection for post-initialization stalls**, specifically keyed to the ~4-hour timeout signature seen in this incident.\n4. **Build a queryable inventory** of VM-runner customers and the versions they're running, to support faster, more targeted response in future incidents and proactive upgrade outreach.",
"postmortem_body_last_updated_at": "2026-11-10T15:02:53.163-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-10T15:02:53.163-08:00",
"reminder_intervals": null,
"resolved_at": "2026-10-28T10:10:38.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/fbvsnn07z92v",
"started_at": "2026-10-28T10:10:38.000-07:00",
"status": "resolved",
"updated_at": "2026-11-10T15:02:53.211-08:00"
},
{
"components": [
{
"created_at": "2023-08-21T10:57:32.631-07:00",
"description": "Harness Continuous Integration Enterprise (CIE) - Self Hosted Runners in Prod 1",
"group_id": "27qq03xf2msg",
"id": "cc654c3j6476",
"name": "Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2023-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-27T15:24:17.476-07:00"
}
],
"created_at": "2026-10-27T12:52:30.529-07:00",
"id": "jmfpjjljc0n3",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-27T15:24:17.519-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T15:24:17.519-07:00",
"id": "nfvqchtxzw13",
"incident_id": "jmfpjjljc0n3",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T15:24:17.519-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-27T15:16:36.592-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T15:16:36.592-07:00",
"id": "wkgl2c4njw6z",
"incident_id": "jmfpjjljc0n3",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T15:16:36.592-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-27T14:15:10.997-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T14:15:10.997-07:00",
"id": "vfk9lx3m5ms0",
"incident_id": "jmfpjjljc0n3",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T14:15:10.997-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "CI pipelines on VM infrastructure are failing at the Initialize step because the task goes to a delegate outside the intended pool",
"created_at": "2026-10-27T12:58:34.228-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T12:58:34.228-07:00",
"id": "x2zv7cmhktrq",
"incident_id": "jmfpjjljc0n3",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T12:58:34.228-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cc654c3j6476",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-27T12:52:30.648-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-27T12:52:30.648-07:00",
"id": "jzgvzwlfdlvm",
"incident_id": "jmfpjjljc0n3",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-27T12:52:30.648-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-27T15:16:36.592-07:00",
"name": "We are currently seeing delegate selection issue for VM infra executions",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "## 1. Summary\n\nOn October 27, 2026, starting around 6:35 AM, customers running CI pipelines on self-managed VM infrastructure \\(a legacy Delegate-based VM runner configuration\\) in Prod1 began experiencing pipeline failures caused by tasks being routed to Delegates that could not actually execute them. The issue affected a small number of customers using this specific, less common configuration \\(VM-based CI without Delegate tags configured\\). It was introduced by a backend deployment and was mitigated by rolling that deployment back; a permanent fix followed.\n\n# 2. Impact\n\n* CI pipelines using VM-based execution without Delegate tags configured began intermittently failing to run on affected Delegates.\n* Other pipeline types and configurations were not affected.\n* Impact was limited to a small number of customers in Prod1 using this specific configuration.\n* The partial outage lasted approximately 8 hours before being resolved via rollback. Affected customers were given a configuration-based workaround \\(setting Delegate tags\\) during the incident.\n\n# 3. Root Cause\n\nOur Delegate platform uses a feature called Capability Checks to determine whether a given Delegate is actually able to execute a specific task, for example, whether it has access to the right infrastructure or connector. When Delegate tags are explicitly configured, this check matters less, but for configurations without tags, it is the primary mechanism for routing work correctly.\n\n\n\nAs part of ongoing work to support capability checking for a newer Delegate version, an optimization was introduced that skipped running a capability check whenever its underlying validation criteria came back empty, the assumption being that an empty criteria value meant there was nothing meaningful to validate, so the Delegate could be treated as already capable.\n\n\n\nIn practice, several existing capability checks had previously harmless defect, which under certain conditions would produce an empty criteria value despite still having a meaningful check that should run. Skipping checks in that state caused Delegates to be automatically treated as capable regardless of whether they actually were and most significantly, for legacy CI VM pipelines, where this logic determines whether a given Delegate is managing the correct virtual machine pool. With the check skipped, Delegates were marked capable of managing any VM pool, which caused CI tasks to be routed to Delegates that could not actually run them.\n\n\n\nBecause the underlying defects were pre-existing but previously harmless, the issue only surfaced once the new optimization began skipping the checks that had been quietly masking them, and it took several hours to appear widely because of a caching layer with a 6-hour refresh interval.\n\n# 4. Resolution\n\n* Identified that CI VM task routing was being driven by faulty capability-check results.\n* Rolled back the backend deployment containing the optimization \\(the first rollback attempt did not fully take effect; a retry succeeded\\).\n* Provided customers with a configuration-based workaround \\(explicitly setting Delegate tags\\) to reduce impact while the rollback was in progress.\n* Reverted the capability-check optimization so checks with empty criteria run normally again rather than being skipped.\n* Added metrics and logging to help identify the specific capability checks that were producing empty criteria incorrectly, so they can be fixed at the source.\n\n# 5. Preventive Actions\n\n1. **Fix the underlying capability checks** that were producing empty criteria incorrectly \\(several affecting CI VM pipelines, with a few additional minor cases identified\\).\n2. **Safely re-introduce the capability-check optimization** once those checks are fixed, since empty criteria should not normally occur once the underlying defects are resolved.\n3. **Add automated test coverage** that exercises this CI VM Delegate-routing use case, since this was a regression that automated tests did not catch.\n4. **Add alerting specific to CI VM infrastructure routing**, since no automated alert fired for this incident and it was first identified by customers.",
"postmortem_body_last_updated_at": "2026-11-10T14:55:40.009-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-10T14:55:40.009-08:00",
"reminder_intervals": null,
"resolved_at": "2026-10-27T15:24:17.519-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/xsr0cq7g8zjm",
"started_at": "2026-10-27T12:52:30.520-07:00",
"status": "resolved",
"updated_at": "2026-11-10T14:55:40.039-08:00"
},
{
"components": [
{
"created_at": "2024-11-07T20:37:07.688-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "rgdxklp8p0kd",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.261-07:00"
},
{
"created_at": "2024-11-07T20:34:14.643-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "pb161tg7ccrs",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.306-07:00"
},
{
"created_at": "2024-11-07T20:38:19.182-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "0kkbcr1h3cvx",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.350-07:00"
},
{
"created_at": "2024-11-07T20:40:44.997-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "trkh53c2bscy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.391-07:00"
},
{
"created_at": "2024-11-07T20:32:37.209-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 1",
"group_id": "27qq03xf2msg",
"id": "n0m4jk53hrym",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.435-07:00"
},
{
"created_at": "2024-11-07T20:33:35.371-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "ctmk1q1dw7nr",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.484-07:00"
},
{
"created_at": "2024-11-07T20:40:12.699-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "th62w3qy4r3m",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.526-07:00"
},
{
"created_at": "2024-11-07T20:41:13.208-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "x3wy7hhj591c",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.579-07:00"
},
{
"created_at": "2026-07-22T22:27:49.688-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 6",
"group_id": "1fpldmj5f61k",
"id": "t8gybmf0b2qt",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.715-07:00"
},
{
"created_at": "2024-11-07T20:39:17.107-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod1",
"group_id": "27qq03xf2msg",
"id": "5znhlv1nwfcf",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.857-07:00"
},
{
"created_at": "2024-11-07T20:39:41.158-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod2",
"group_id": "qbxn0mh45jhv",
"id": "qvs27pfch8vy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.900-07:00"
},
{
"created_at": "2026-07-22T22:28:26.126-07:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod6",
"group_id": "1fpldmj5f61k",
"id": "q1369hjq7nn4",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.939-07:00"
}
],
"created_at": "2026-10-19T02:19:47.725-07:00",
"id": "9734l1pk9khw",
"impact": "maintenance",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "The scheduled maintenance has been completed.",
"created_at": "2026-10-23T23:32:06.425-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T23:32:06.425-07:00",
"id": "gw732td06qkl",
"incident_id": "9734l1pk9khw",
"status": "completed",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T23:32:06.425-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "under_maintenance"
}
],
"body": "Verification is currently underway for the maintenance items.",
"created_at": "2026-10-23T23:20:49.005-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T23:20:49.005-07:00",
"id": "yv6bvddnf3wl",
"incident_id": "9734l1pk9khw",
"status": "verifying",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T23:20:49.005-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "under_maintenance",
"old_status": "operational"
}
],
"body": "Scheduled maintenance is currently in progress. We will provide updates as necessary.",
"created_at": "2026-10-23T22:36:51.565-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-23T22:36:51.565-07:00",
"id": "xdsh4q3vk6v7",
"incident_id": "9734l1pk9khw",
"status": "in_progress",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T22:36:51.565-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We're upgrading our CI infrastructure to improve reliability and security.\n\n Expected impact:\n - Brief connectivity blips as we migrate outbound traffic.\n - Builds running during the migration may fail on outbound calls\n (git clone, artifact pulls, webhooks) and will need a re-run.\n - Re-runs will succeed once the migration completes.\n\n Your action:\n If you haven't already allowlisted the new egress IP ranges\n (8.235.0.0/24 and 34.33.12.0/24), please do so now.",
"created_at": "2026-10-19T02:19:47.855-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T02:19:47.855-07:00",
"id": "s5fw2kdhw7h7",
"incident_id": "9734l1pk9khw",
"status": "scheduled",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T02:19:47.855-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-23T23:20:49.005-07:00",
"name": "Planned maintenance: Harness CI egress IP migration",
"page_id": "zmnt6tkys0q0",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-23T23:32:06.425-07:00",
"scheduled_auto_completed": true,
"scheduled_auto_in_progress": true,
"scheduled_for": "2026-10-24T22:30:00.000-07:00",
"scheduled_remind_prior": true,
"scheduled_reminded_at": null,
"scheduled_until": "2026-10-24T23:30:00.000-07:00",
"shortlink": "https://stspg.io/6sm9tqxjs3sl",
"started_at": "2026-10-19T02:19:47.674-07:00",
"status": "completed",
"updated_at": "2026-10-23T23:32:06.470-07:00"
},
{
"components": [
{
"created_at": "2024-07-10T21:08:30.607-07:00",
"description": "Global Gateway intermittently encountering connection errors in Prod 4",
"group_id": "x1089mr22815",
"id": "p9t9g01vgy47",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 1,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.045-07:00"
},
{
"created_at": "2024-07-10T21:11:25.979-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Self Hosted Runners in Prod 4",
"group_id": "x1089mr22815",
"id": "bp1hsfllmds0",
"name": "Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"page_id": "zmnt6tkys0q0",
"position": 4,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.069-07:00"
},
{
"created_at": "2024-07-10T21:10:55.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 4",
"group_id": "x1089mr22815",
"id": "czhk96jtx93b",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.784-07:00"
},
{
"created_at": "2024-11-07T20:37:07.688-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "rgdxklp8p0kd",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.261-07:00"
},
{
"created_at": "2024-11-07T20:40:44.997-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "trkh53c2bscy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.391-07:00"
},
{
"created_at": "2024-07-10T21:12:38.445-07:00",
"description": "Harness Security Testing Orchestration (STO) in Prod 4",
"group_id": "x1089mr22815",
"id": "k0hbqz5c81v8",
"name": "Security Testing Orchestration (STO)",
"page_id": "zmnt6tkys0q0",
"position": 10,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.164-07:00"
},
{
"created_at": "2024-07-10T21:20:08.616-07:00",
"description": "Harness Infrastructure as Code Management (IaCM) in Prod 4",
"group_id": "x1089mr22815",
"id": "nxsthnf9gk3g",
"name": "Infrastructure as Code Management (IaCM)",
"page_id": "zmnt6tkys0q0",
"position": 14,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.188-07:00"
},
{
"created_at": "2025-08-27T10:29:44.922-07:00",
"description": "Core platform constructs eg. Platform delegate (eg. Delegate communication), Platform Core (eg. Account, Login, Project/Org, Connectors, Secrets, Delegate, Settings, Notifications, Audits, Templates, Services, Environments , Policies, File Store, Log Uploads), Platform Authentication \u0026 Authorization (login, RBAC, logout, API keys, Service account, Tokens etc.)",
"group_id": "x1089mr22815",
"id": "561g9b7sqgct",
"name": "Platform",
"page_id": "zmnt6tkys0q0",
"position": 17,
"showcase": true,
"start_date": "2025-08-27T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-19T01:04:11.212-07:00"
}
],
"created_at": "2026-10-19T00:46:51.821-07:00",
"id": "9xpkprn54m6z",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-19T01:21:25.895-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:21:25.895-07:00",
"id": "tgzqsf2538kw",
"incident_id": "9xpkprn54m6z",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:21:25.895-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-19T01:04:11.251-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:04:11.251-07:00",
"id": "912js2k93ds5",
"incident_id": "9xpkprn54m6z",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:04:11.251-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-19T01:02:50.060-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:02:50.060-07:00",
"id": "3zwtk30334mw",
"incident_id": "9xpkprn54m6z",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:02:50.060-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-19T01:02:29.637-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T01:02:29.637-07:00",
"id": "jj7zlhzsgn6h",
"incident_id": "9xpkprn54m6z",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T01:02:29.637-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p9t9g01vgy47",
"name": "Prod 4 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "bp1hsfllmds0",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Self Hosted Runners",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "k0hbqz5c81v8",
"name": "Prod 4 - Security Testing Orchestration (STO)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "nxsthnf9gk3g",
"name": "Prod 4 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "561g9b7sqgct",
"name": "Prod 4 - Platform",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-19T00:46:52.239-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-19T00:46:52.239-07:00",
"id": "j508529yg8js",
"incident_id": "9xpkprn54m6z",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-19T00:46:52.239-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-19T01:02:29.637-07:00",
"name": "Degraded performance on prod4",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "## **Summary**\n\nOn October 19, 2026, between approximately 12:10 AM and 12:55 AM PDT, customers on Prod4 experienced errors and intermittent unavailability of Harness Pipelines services. Heavy pipeline executions drove a sharp rise in CPU and memory use on the pipeline service, and for a short period all service instances were unavailable. Harness engineering scaled the service both vertically and horizontally, which restored service. A fix for the underlying behavior is being rolled out.\n\n## **Root Cause**\n\nOn October 19th, high number of concurrent executions with very large execution graphs drove a sharp increase in cluster resources. The service stores and retrieves this full execution graph for visualization purposes and further progressing the pipelines to next states. Our auto-scalaers on Prod4 took longer than usual for scaling resources due to a sharp rise in CPU and memory. \n\nAs a part of the fix, the execution graph update will now process graph changes incrementally instead of the full graph. It is being rolled out in a phased manner and will be completely rolled out at the earliest.\n\n## **Impact**\n\n* Affected users: Customers on Prod4.\n* Symptom: unavailability of pipeline functionality in the Harness platform.\n* Data: No data loss.\n* Suggested workaround: None required from customers. Harness engineering resolved the issue.\n* Duration of impact: Approximately 20 minutes of full unavailability and degraded performance for 25 minutes. Harness continued monitoring after recovery and confirmed platform stability with customers.\n\n## **Remediation**\n\n**Immediate**\n\nHarness engineering scaled the pipeline service both vertically \\(more resources per instance\\) and horizontally \\(more instances\\), and removed instances in an unhealthy state. The instance count stabilized and normal service was restored. The system will stay scaled up till the fix is completely rolled out. \n\n**Permanent fix and Action Items**\n\nTo prevent such issues from happening again, Harness is\n\n1. Tuning the auto heal/scaling process so that impact is mimimal to none.\n2. Optimizing the execution graph flow so that large graphs can be process faster.",
"postmortem_body_last_updated_at": "2026-10-23T11:24:10.455-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-23T11:24:10.455-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-19T01:21:25.895-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/bmvp2852wlrr",
"started_at": "2026-10-19T00:46:51.798-07:00",
"status": "resolved",
"updated_at": "2026-10-23T11:24:10.479-07:00"
},
{
"components": [
{
"created_at": "2022-03-15T17:37:45.978-07:00",
"description": "Harness Continuous Delivery Next Generation (CDNG) in Prod 3",
"group_id": "z7txn2324j6k",
"id": "l1sx8bck1yzb",
"name": "Continuous Delivery - Next Generation (CDNG)",
"page_id": "zmnt6tkys0q0",
"position": 2,
"showcase": true,
"start_date": "2021-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-15T03:27:52.167-07:00"
}
],
"created_at": "2026-10-15T01:04:14.381-07:00",
"id": "xdt6vk6q7k5k",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-15T01:15:44.044-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T01:15:44.044-07:00",
"id": "df7fdxc61y5d",
"incident_id": "xdt6vk6q7k5k",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T01:15:44.044-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-15T01:09:35.314-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T01:09:35.314-07:00",
"id": "76sy7n6k44lw",
"incident_id": "xdt6vk6q7k5k",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T01:09:35.314-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-15T01:05:07.284-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T01:05:07.284-07:00",
"id": "2yyjbx0mmy2z",
"incident_id": "xdt6vk6q7k5k",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T01:05:07.284-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "l1sx8bck1yzb",
"name": "Prod 3 - Continuous Delivery - Next Generation (CDNG)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-15T01:04:14.652-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T01:04:14.652-07:00",
"id": "31f80hzql4y6",
"incident_id": "xdt6vk6q7k5k",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T01:04:14.652-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-15T01:09:35.314-07:00",
"name": "Degraded performance in UI execution graph in prod3",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "# Root Cause Analysis Report\n\n# 1. Summary\n\nOn October 15, 2026, between 12:24 AM and 1:16 AM PT, Prod3 experienced a delay in how quickly execution graphs, aggregations, and observability data updated. The underlying workflow execution engine continued to function normally throughout and note that, this was a data-freshness and UI-responsiveness issue, not a workflow execution failure. The root cause was a backend search-service dependency in our event-processing pipeline that became a bottleneck under an unusually high volume of API calls to a specific execution-details endpoint, concentrated from one customer account. We mitigated by scaling the search cluster, and after a brief recurrence later that morning, doubled its capacity. We subsequently shipped a permanent fix that removed the search-service dependency from this code path entirely, cutting search load by about 95%.\n\n# 2. Impact\n\n* Prod3 users may have seen delayed or stale execution graphs, aggregation views, and observability data for about 52 minutes, with a brief recurrence later the same morning.\n* Underlying workflow execution was not affected and pipelines continued to run and complete normally throughout.\n* No confirmed workflow execution failures occurred, and no data was lost.\n* We temporarily applied protective rate-limiting safeguards on the affected API and coordinated with the customer whose traffic drove the load while a permanent fix was developed and deployed.\n\n# 3. Root Cause\n\nOur platform processes workflow execution events through an internal, queue-based pipeline. A step in that pipeline performed a synchronous search-service lookup as part of completing each event which is aa dependency that, under normal conditions, added negligible delay.\n\n\n\nAn unusually high and sustained volume of calls to an execution-details API, concentrated from one customer account, drove a large increase in search-service query volume. This pushed CPU on the search cluster's warm-tier nodes to saturation, which in turn slowed the synchronous lookups in the event-processing pipeline, causing events to back up and execution-graph and observability data to fall behind in near-real time, even though underlying workflow execution did not depend on this same path.\n\n# 4. Resolution\n\n* Verified core databases and recent deployments were not responsible, narrowing the investigation to the search-service dependency.\n* Identified and mitigated warm-tier search-cluster CPU saturation by scaling cluster capacity during the incident.\n* Further increased search-cluster capacity after a brief recurrence later the same morning.\n* Added temporary logging to better identify which traffic was driving load, and worked with the customer to understand their usage pattern.\n* Applied temporary protective rate-limiting on the affected endpoint as a safeguard.\n* Shipped a permanent API optimization that removes the search-service dependency from this workflow-event-processing path, which reduced search load by about 95%.\n\n# 5. Preventive Actions\n\n1. **Add duration-based alerting** for sustained search-cluster CPU saturation, correlated with event-processing lag, so prolonged stress escalates automatically rather than depending on manual observation.\n2. **Validate search-cluster capacity headroom** going forward, and evaluate whether capacity can now be safely reduced given the dependency has been removed from the hot path.\n3. **Decouple or bound similar dependencies**: audit other workflow-event-processing paths for synchronous search-service calls and inventory similar dependencies elsewhere in the platform.\n4. **Add finer-grained rate limiting** on the affected endpoint so that one customer's traffic pattern cannot affect other customers sharing the same infrastructure.\n5. **Define a clearer policy** for applying protective traffic controls when a single customer's usage threatens shared-platform stability.\n\n\n\n\n\n_We appreciate your patience while we worked through a complete resolution, and we remain committed to strengthening the resilience of our platform as a result of this incident._",
"postmortem_body_last_updated_at": "2026-11-10T14:26:41.609-08:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-11-10T14:26:41.609-08:00",
"reminder_intervals": null,
"resolved_at": "2026-10-15T01:15:44.044-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/8q9r632ktw3c",
"started_at": "2026-10-15T01:04:14.373-07:00",
"status": "resolved",
"updated_at": "2026-11-10T14:26:41.640-08:00"
},
{
"components": [
{
"created_at": "2024-04-17T02:36:03.213-07:00",
"description": "Harness Infrastructure as Code Management (IaCM) in Prod1",
"group_id": "27qq03xf2msg",
"id": "15bv7h1knvsk",
"name": "Infrastructure as Code Management (IaCM)",
"page_id": "zmnt6tkys0q0",
"position": 15,
"showcase": true,
"start_date": "2024-01-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-12T18:56:07.012-07:00"
}
],
"created_at": "2026-10-12T18:06:13.615-07:00",
"id": "20bc41tsx8rz",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-12T21:29:35.168-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-12T21:29:35.168-07:00",
"id": "x4hf8d89h2yg",
"incident_id": "20bc41tsx8rz",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T21:29:35.168-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-12T18:56:07.210-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-12T18:56:07.210-07:00",
"id": "h98y6qvv1s6w",
"incident_id": "20bc41tsx8rz",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T18:56:07.210-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "The issue has been identified and a fix is being implemented.",
"created_at": "2026-10-12T18:07:25.351-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-12T18:07:25.351-07:00",
"id": "lsh49l5zy6r8",
"incident_id": "20bc41tsx8rz",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T18:07:25.351-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "15bv7h1knvsk",
"name": "Prod 1 - Infrastructure as Code Management (IaCM)",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently investigating this issue.",
"created_at": "2026-10-12T18:06:13.722-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-12T18:06:13.722-07:00",
"id": "2gqhv0hv98xb",
"incident_id": "20bc41tsx8rz",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T18:06:13.722-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-12T18:56:07.210-07:00",
"name": "Prod1: Intermittent issue with IACM stages in some pipelines",
"page_id": "zmnt6tkys0q0",
"postmortem_body": ". \n\n.",
"postmortem_body_last_updated_at": "2026-10-22T20:16:42.348-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-22T20:16:42.348-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-12T21:29:35.168-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/zygklrx86y2b",
"started_at": "2026-10-12T18:06:13.604-07:00",
"status": "resolved",
"updated_at": "2026-10-22T20:16:42.373-07:00"
},
{
"components": [
{
"created_at": "2024-07-10T21:10:55.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 4",
"group_id": "x1089mr22815",
"id": "czhk96jtx93b",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 5,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.784-07:00"
},
{
"created_at": "2022-03-15T18:09:19.330-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 3",
"group_id": "z7txn2324j6k",
"id": "4fpjqbv7k6fp",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2021-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.833-07:00"
},
{
"created_at": "2024-09-12T17:05:44.740-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod Eu1",
"group_id": "6y3np02bqtxf",
"id": "y2wg26xx8r7r",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-12T12:31:45.914-07:00"
},
{
"created_at": "2024-11-07T20:37:07.688-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "rgdxklp8p0kd",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 6,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.261-07:00"
},
{
"created_at": "2021-06-01T14:48:02.147-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 1",
"group_id": "27qq03xf2msg",
"id": "kzjygtpl015t",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-04-10T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.883-07:00"
},
{
"created_at": "2021-06-01T14:56:05.591-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "hp77w1mwh2xg",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2021-04-10T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-31T22:27:44.931-07:00"
},
{
"created_at": "2024-11-07T20:34:14.643-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "pb161tg7ccrs",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.306-07:00"
},
{
"created_at": "2024-11-07T20:38:19.182-08:00",
"description": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "0kkbcr1h3cvx",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.350-07:00"
},
{
"created_at": "2024-11-07T20:40:44.997-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod4",
"group_id": "x1089mr22815",
"id": "trkh53c2bscy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2024-07-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.391-07:00"
},
{
"created_at": "2026-07-22T22:26:59.344-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Mac Cloud Builds in Prod 6",
"group_id": "1fpldmj5f61k",
"id": "8mjkl07z9kn1",
"name": "Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 7,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-12T12:31:46.093-07:00"
},
{
"created_at": "2024-11-07T20:32:37.209-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 1",
"group_id": "27qq03xf2msg",
"id": "n0m4jk53hrym",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.435-07:00"
},
{
"created_at": "2024-11-07T20:33:35.371-08:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 2",
"group_id": "qbxn0mh45jhv",
"id": "ctmk1q1dw7nr",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.484-07:00"
},
{
"created_at": "2024-11-07T20:40:12.699-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod3",
"group_id": "z7txn2324j6k",
"id": "th62w3qy4r3m",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2023-12-01T00:00:00.000-08:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.526-07:00"
},
{
"created_at": "2024-11-07T20:41:13.208-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in EU1",
"group_id": "6y3np02bqtxf",
"id": "x3wy7hhj591c",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2024-09-12T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.579-07:00"
},
{
"created_at": "2026-07-22T22:27:49.688-07:00",
"description": "Harness Continuous Integration Enterprise(CIE) - Windows Cloud Builds in Prod 6",
"group_id": "1fpldmj5f61k",
"id": "t8gybmf0b2qt",
"name": "Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 8,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.715-07:00"
},
{
"created_at": "2024-11-07T20:39:17.107-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod1",
"group_id": "27qq03xf2msg",
"id": "5znhlv1nwfcf",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.857-07:00"
},
{
"created_at": "2024-11-07T20:39:41.158-08:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod2",
"group_id": "qbxn0mh45jhv",
"id": "qvs27pfch8vy",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2023-11-01T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.900-07:00"
},
{
"created_at": "2026-07-22T22:28:26.126-07:00",
"description": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds in Prod6",
"group_id": "1fpldmj5f61k",
"id": "q1369hjq7nn4",
"name": "Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"page_id": "zmnt6tkys0q0",
"position": 9,
"showcase": true,
"start_date": "2026-07-22T00:00:00.000-07:00",
"status": "operational",
"updated_at": "2026-10-23T23:20:48.939-07:00"
}
],
"created_at": "2026-10-12T08:25:16.865-07:00",
"id": "th34d9m8qyc5",
"impact": "minor",
"impact_override": null,
"incident_updates": [
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "y2wg26xx8r7r",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "8mjkl07z9kn1",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This incident has been resolved.",
"created_at": "2026-10-12T12:31:46.632-07:00",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-12T09:31:46.000-07:00",
"id": "drj45wj21b25",
"incident_id": "th34d9m8qyc5",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T12:32:27.621-07:00",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "kzjygtpl015t",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "n0m4jk53hrym",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "5znhlv1nwfcf",
"name": "Prod 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "hp77w1mwh2xg",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "ctmk1q1dw7nr",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "qvs27pfch8vy",
"name": "Prod 2 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "4fpjqbv7k6fp",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "pb161tg7ccrs",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "th62w3qy4r3m",
"name": "Prod 3 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "czhk96jtx93b",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "rgdxklp8p0kd",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "trkh53c2bscy",
"name": "Prod 4 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "y2wg26xx8r7r",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "0kkbcr1h3cvx",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "x3wy7hhj591c",
"name": "Prod Eu 1 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "8mjkl07z9kn1",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Mac Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "t8gybmf0b2qt",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Windows Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
},
{
"code": "q1369hjq7nn4",
"name": "Prod 6 - Continuous Integration Enterprise(CIE) - Linux Cloud Builds",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are currently experiencing HTTP 429 (rate-limiting) errors when Harness Cloud builds access Maven Central, which may impact build execution. Our team is actively working with Sonatype to resolve the issue as quickly as possible. We will provide further updates as we make progress.",
"created_at": "2026-10-12T08:25:17.707-07:00",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-12T08:25:17.707-07:00",
"id": "z8syp51n6lv7",
"incident_id": "th34d9m8qyc5",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-12T08:25:17.707-07:00",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Harness CI Cloud build issues when accessing Maven Central",
"page_id": "zmnt6tkys0q0",
"postmortem_body": "tldr: \\(There was a brief rate limiting by maven central\\) \n\nMore details to follow",
"postmortem_body_last_updated_at": "2026-10-22T20:07:16.060-07:00",
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-22T20:07:16.060-07:00",
"reminder_intervals": null,
"resolved_at": "2026-10-12T09:31:46.000-07:00",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/ffnvmypvjnqh",
"started_at": "2026-10-12T12:31:46.000-07:00",
"status": "resolved",
"updated_at": "2026-10-22T20:07:16.085-07:00"
}
],
"page": {
"id": "zmnt6tkys0q0",
"name": "Harness",
"url": "https://status.harness.io"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}