Snapshot 157511
Raw (normalized text)
Scripts and page chrome removed; this is what change detection compares.
{
"components": [
{
"components": [
"y79r86xbdvqs",
"7fp5ydxz32r2",
"lb05w3r7wnp6",
"4c7pnjqt4rw1",
"23pr4cv0x77q",
"fgj568lsbfvr",
"2g7228rmhgk7",
"ncrpjwb1dl2j",
"65j17f6lbqf5",
"480hb77jpwvq",
"ypxsr0n3r72x",
"brzjkmc9ccky",
"0f8st0crv2jr",
"k12vm7h1vhkj",
"zjfr64ky6n65",
"kd67xhc496mq",
"r6rl2csmxyrz",
"sp3x5zz6q3n0",
"ct2ycwn7t9v0",
"53q852hbcfn9",
"913kv3whf0n9",
"5txf886jydqg",
"ddp76dct0ft8",
"fjjlg8hd3vr0",
"j99ypph15vgh",
"xz3mpdh89145",
"7206tkwxcw9y",
"gcm3m5734d2m",
"858pch44wyr0",
"ln34lcy4mlzl",
"qll1msg3x091",
"00gln12pq3h4",
"7yp4fv7nrcnz"
],
"created_at": "2026-03-01T05:02:12.991Z",
"description": null,
"group_id": null,
"id": "bt4fyv3z54lv",
"name": "European Union",
"page_id": "rtthtlr99830",
"position": 31,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-01T05:02:12.991Z"
},
{
"components": [
"l6dspfl85f3r",
"8c2g6j81glkx",
"n56ry3p5l4hc",
"lcrmj7bnmcww",
"v15xk3zm83z7",
"mpt9g0rfd06q",
"npkg31hnqw13",
"846j2gs041rk",
"bsnhnz1scnpj",
"y7mw6n7zgjzc",
"wzpb1n3g2g5r",
"4knfzrmmkxcm",
"r7lgf82h087c",
"vg3pxqh3rdpz",
"tr2qsl9wf2s6",
"595lcfcm5w04",
"gtc1c87pl29w",
"j6p5vv4572jc",
"8nf2cb758g0k",
"z8351chcxsms",
"cg5fxshsbm22",
"67bf9m7nw5sm",
"l4cb75n868sj",
"qdtw9mtr57wv",
"3dxdt7wg53st",
"tg7f8zhf7yz0",
"mjgl6bcsw3nh",
"2h3vc1w1lkyn",
"ljxlbf9bfsmm",
"bbvbwjlkdhm6",
"41wlh3ntkhcv",
"8336pnryxtbv",
"nk44pk6nmj69"
],
"created_at": "2026-03-01T05:03:07.150Z",
"description": null,
"group_id": null,
"id": "8md89tvvp0jc",
"name": "United States",
"page_id": "rtthtlr99830",
"position": 62,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:28.878Z"
},
{
"components": [
"q3659hxv43n1",
"cpvmj871ptxl",
"krn39gwdd70k",
"3nzpjjjn82xy",
"zzb0pmw7ycqp",
"8gc73k6qlb6s",
"5j9zynsctq23",
"vbx04xq0xxdn",
"hpd3qykjd1kc",
"nk0wqq99bm5l",
"tr7rvtdbkx2j",
"77p7q6pw5zf5",
"g6m2nv651z7v",
"3g44sf4mk0gd",
"7hf16tg7gwhs",
"m149fzmchj41",
"2xgkcw3k95j8",
"rj0fm6x837p1",
"vm9nsmnz3dz8",
"l7z463x0yc4p",
"3q51s33tt4vw",
"fmh2jgkzlw19",
"41smwq7585sq",
"yxvtzxjcl3tv",
"9bs2hkv4z1j3",
"zn093v9pvbqy",
"jvls0hs1h4vq",
"bgqc5w4r0kkz",
"p37x9ps3bj1z",
"w2rmcymky47q",
"c2x2ghbk65yz"
],
"created_at": "2026-03-01T05:03:55.947Z",
"description": null,
"group_id": null,
"id": "q9rqbbwgxqfz",
"name": "Japan",
"page_id": "rtthtlr99830",
"position": 93,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:30.718Z"
},
{
"components": [
"z3sy0vyjw5nh",
"l6ks1wt87fkn",
"vb3gj33wg3k7",
"ypr5s4q1hm75",
"mm15d5qhmjs4",
"h8gqgzz3h2hm",
"300hjmlxvrjd",
"1tzz40lpcr45",
"d2h6hy597bm7",
"8l2tmng27f38",
"fdbs58d68mzf",
"62zfjzqjw2x9",
"k9cmpc2dj6jc",
"917hfvqjwj8x",
"3k4b8dwz0608",
"wrtmj4dtl705",
"3vprkgssdlh4",
"2l1vvr35m7qv",
"6qcvzf9p990y",
"wv0q1gmp017j",
"cndlfd0h95qg",
"ps1d17kspjt1",
"rqnswks3x9fk",
"st931vtzs3sr",
"d100td92x61b"
],
"created_at": "2026-03-01T05:04:45.624Z",
"description": null,
"group_id": null,
"id": "k33cxmfzs4sq",
"name": "India",
"page_id": "rtthtlr99830",
"position": 124,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:32.380Z"
},
{
"components": [
"xbykvxmjnrk2",
"84y35rwdwqgv",
"8zljfv2sfzsl",
"7w7fysf6r6my",
"wnvtrmty5j3s",
"ptdpxr46y9pn",
"f886g08c5wcg",
"3d4b3t9gxx38",
"zc68ck0kb3d3",
"18wy9p125vwk",
"fbqmb4cj5l8s",
"6qsnlxxd580v",
"4kgd8kcp30yc",
"d7d158n56ljl",
"zkkw1zzyfl9g",
"xqngmly4vgx1",
"kyrf6l3czdwt",
"yxqx6rk7r5pb",
"4wm6clmp1cq6",
"183rm9nh3kyk",
"yq1r832kt8rm",
"cdffsj02gkcz",
"l5rlsr6cp7hx",
"pqwx0xn08kr5",
"dspkt4m2p068",
"9fqy9d8yw662",
"189vycq36r6m"
],
"created_at": "2026-03-01T05:05:36.883Z",
"description": null,
"group_id": null,
"id": "1bk483dn7k09",
"name": "Singapore",
"page_id": "rtthtlr99830",
"position": 155,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:33.800Z"
},
{
"components": [
"gpw6tys3clwc",
"07mgmfr738cj",
"z4jjbgy74szq",
"dvykgswf2k0s",
"c6rhl9ll21zc",
"mftdt1b5ywls",
"yp3pcr2dxddc",
"jl2v6mvd5b6k",
"x77dpw7qv1s8",
"gw7mzbg89tnn",
"sfj6hl63qgyv",
"llwh8qz8h5rr",
"dqzy1z7qdsjq",
"v81srf170wcz",
"8xsxpp9l2lgb",
"m8dfg4nnsnnl",
"p823pmtmf6ts",
"m5s1zxk8kmn8",
"6j0bgjbv7hwv",
"1p37tmtsj0rc",
"3c8cp8hpwwfq",
"mycc6kpqt3fn",
"p1xxg9yp99gy",
"pyks6vlqjmq4",
"qpf0svb6wxc1",
"rffr6f31ycmh",
"2n16xr60pkhs",
"ns9px4vr95jr",
"8hwwdhcr1dr3",
"b588ylzw2fl8"
],
"created_at": "2026-03-01T05:06:24.755Z",
"description": null,
"group_id": null,
"id": "0h50tbzgz6n8",
"name": "Australia",
"page_id": "rtthtlr99830",
"position": 186,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:35.219Z"
},
{
"components": [
"qyfggkswtl6d",
"xq9cgtr2j1lx",
"qwqd9l6wd99s",
"lt0tpx5k79yw",
"h3lgy01wdmkp",
"mh7lnjbw9qm7",
"nvjlr93rnjc4",
"kz1q64tct4zd",
"c9xm92nq42rr",
"nf1bdvprbpk9",
"mwyq71qp147r",
"b530c94j69sn",
"g8jbsw9ggslw",
"8f58ltqwc6d5",
"rlshxj209kkw",
"pcm9s88dbm4w",
"sts7j63f9yrg",
"1cz70c9jc9g3",
"4jkmxs1hv7zt",
"k2m3fbh6d707",
"vtkzq54ddhdx",
"k9lyqrwrdlfr",
"q3z58ss6hxq0",
"w1kll2gbb0q0",
"50pwf5rhgsjq",
"w0l5dmd6nqb4",
"f1h5pl11x8zz",
"9544sds4d80k"
],
"created_at": "2026-03-01T05:07:13.260Z",
"description": null,
"group_id": null,
"id": "3w1cgj3h4gyt",
"name": "Canada",
"page_id": "rtthtlr99830",
"position": 217,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:36.678Z"
},
{
"components": [
"27ctv2wpdcfg",
"h6zz0q3n84b2",
"94dcphpczw7c",
"gmdl3098b7rs",
"9kscfq55m694",
"783kn4wxg02s",
"6t2q1lmjdj77",
"qms1qpcm45nv",
"zmq4bq4lvhls",
"xbdgkb2qdb8g",
"xxgljl1jww6l",
"km9qcv5sn1yd",
"tzqgwq2rhxxd",
"43tsnchcwx4c",
"4x4zp0w3rr8w",
"hs0rww4msz60",
"gh73lnyjsyby",
"jhv3yk5xwng1",
"lgjh5ml6hbqp",
"6zr6gjvqmsn7",
"3v76qcl5vs6g",
"g616ydxnz8mm",
"zhswm6ys2v05",
"52s2s89rfp1p",
"x643741pxvfv",
"9c0f5nsqmlzn",
"bvtmf7t1n17s",
"rszv69kjxlqx"
],
"created_at": "2026-03-01T05:08:05.626Z",
"description": null,
"group_id": null,
"id": "sfkr8z0m2j3k",
"name": "UK",
"page_id": "rtthtlr99830",
"position": 248,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:38.141Z"
},
{
"components": [
"2ygh23ff8h6m",
"yxrnldr82n28",
"8d7c1tbw9pwf",
"nvc1f3sym3vt",
"q6kgcjjspk87",
"zvsg4rh1mnwt",
"wrgt5jfp0209",
"8sbycpy97cqq",
"h5dzkjpzrvjq",
"3pnmqg8dk53c",
"w97ws5wv8w9p",
"38dh0ylcgdsp",
"csxtm3h7x4n9",
"0wt94x63jwtv",
"80c46ftnm4ml",
"jzzp956r6s66",
"12fkf0pyp76s",
"rsydfqj2c0hq",
"d14mv7ytvfcs",
"lscp0rm7l1hl",
"z81lyq17tcpd",
"3w1m73wx6z5w",
"4xhwx1gc1dp4",
"4j27vn017yl2",
"79pjl71vdzph",
"jlh6n7yqgvn4",
"tkb8f3wcwxy5",
"c86z2hz2tq4g",
"c7m5pjy5dlt9"
],
"created_at": "2026-03-01T05:09:46.756Z",
"description": null,
"group_id": null,
"id": "qt4tq8wc76h0",
"name": "Switzerland",
"page_id": "rtthtlr99830",
"position": 310,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:41.898Z"
},
{
"components": [
"5r7y82jcck86",
"jtw5pmqtprky",
"65q6pc893qcb",
"n98d46b5jln4",
"gq9q3fcbwtfk",
"rhn2bx02dxb2",
"df1ks4zv3v3l",
"zjyym1tlym73",
"552m89zzqqk5",
"hxq6rjpnjn9p",
"8hhbgkkkhv00",
"qvgl34cjv1wq",
"b6np20vdchdf",
"7kcjn4ln8xn9",
"r4rfd833wnpp",
"5h25rdk78gbh",
"yswh8f5ybxl5",
"6ty2nd8j32nf",
"wrclpn7v9hmp",
"9d2lxvq313sg",
"0xbghc66vww7",
"mhhmzrpxq1tf",
"w4c3bjn1tspf",
"s0hdzrryp1xh",
"gzjhx51s0pnk",
"bs542fg31xgz",
"8zj0q2h51fwk",
"hzcphlfzgd5c",
"r2v5lxv3b441",
"hkbr7kz9c650"
],
"created_at": "2026-03-01T05:10:35.405Z",
"description": null,
"group_id": null,
"id": "hdlv8jj8xx7k",
"name": "Delayed US",
"page_id": "rtthtlr99830",
"position": 341,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:43.798Z"
},
{
"components": [
"j2q9skgwpkxz",
"j8n8snksxrs2",
"n9ntvk3mq2cf",
"y27l4v5ysgml",
"s7vfb3bp23bf",
"4r505z24x0hf",
"x74pds8zch9x",
"4547xj65sv7h",
"9llwt0qrqv2d",
"36ytksccyjbj",
"dzykrqn5z0w3",
"4xpl9rccqbr4",
"ftqqgvpdxjvx",
"gfhklcs75j3p"
],
"created_at": "2026-03-01T05:11:26.444Z",
"description": null,
"group_id": null,
"id": "xqvsw9dkctvb",
"name": "Delayed EU",
"page_id": "rtthtlr99830",
"position": 372,
"showcase": true,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-02T05:10:45.346Z"
},
{
"components": [
"7spvxn33vk0z",
"hsq8shlw8x96",
"1qyn591w9g4z",
"swpjlh4qc9l1",
"yg8wh76q4gvq",
"d0lv3z305j78",
"whyl9zg6s1sp",
"7flzt357rnyp",
"lnqgzkz4tf5j",
"wbgjq3418c1v",
"vxrql1bzrrvc",
"sy66ljhtwsv8",
"1wf1xmhrhq7s",
"ymhpprrvvhcf",
"thkfdn1c5mfg",
"zfj8gjvvqt3d",
"zn7gvt8c98f0",
"mm35ndb6s43c",
"tgw0r1kfpy1k",
"hgkf0xtzyp8c",
"rxyzcblcdks2",
"vq7wkbhmx9tt",
"35m444hvvjcy",
"zt05dpjx5ytw"
],
"created_at": "2026-03-10T03:46:32.537Z",
"description": null,
"group_id": null,
"id": "m1b1ynftp3sv",
"name": "UAE",
"page_id": "rtthtlr99830",
"position": 374,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-03-10T03:46:32.537Z"
},
{
"components": [
"djbx3g7f1p4m",
"vn7zp4n3mzj6",
"s5fqcv19k46r",
"50tfcjd03c99",
"qv3dvb26ts0g",
"gjhhgzp8b58p",
"f39m1z04zd2r",
"byd70k5q2fws",
"tcfsxntwjgm7",
"jtq08fp5wh7j",
"qm47gq7vkcbc",
"6zxdzgk6q29y",
"k96tf4g0757m",
"z8b12d0dnbvl",
"s9ssyj3qvjmx",
"ct30dytndsf9",
"j8lh27bbz2qh",
"zfbq1pt5j7hc",
"41nzjqkhlf1r",
"gzry77y3y7cd",
"4v498zb7n9qz",
"mlhgy0q2tbxr"
],
"created_at": "2026-07-07T01:57:14.586Z",
"description": null,
"group_id": null,
"id": "2r9tpytl7r6r",
"name": "South Korea",
"page_id": "rtthtlr99830",
"position": 375,
"showcase": false,
"start_date": null,
"status": "operational",
"updated_at": "2026-07-07T01:57:14.586Z"
}
],
"incidents": [
{
"components": [],
"created_at": "2026-10-23T19:40:00.000Z",
"id": "cgpr7x2b39vl",
"impact": "none",
"impact_override": "none",
"incident_updates": [
{
"affected_components": null,
"body": "Between 20:42 and 21:19 UTC on October 23, some Orchestrator customers in Europe saw slow responses and intermittent errors. This included robot calls to get assets, process queue items and report job status. Some unattended jobs may have failed during this time. The cause was a sudden increase in load that reached the database’s resource limits. We added database capacity, and performance has been back to normal since 21:19 UTC. Customers whose jobs failed in this window may need to restart them.",
"created_at": "2026-10-23T21:35:48.346Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-23T19:40:00.000Z",
"id": "gg306mmfz40l",
"incident_id": "cgpr7x2b39vl",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-23T21:35:48.346Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Degraded performance and intermittent failures in Orchestrator (Europe)",
"page_id": "rtthtlr99830",
"postmortem_body": null,
"postmortem_body_last_updated_at": null,
"postmortem_ignored": false,
"postmortem_notified_subscribers": false,
"postmortem_notified_twitter": false,
"postmortem_published_at": null,
"reminder_intervals": null,
"resolved_at": "2026-10-23T19:40:00.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/gxndr7frwg6n",
"started_at": "2026-10-23T21:35:48.280Z",
"status": "resolved",
"updated_at": "2026-10-23T21:35:48.354Z"
},
{
"components": [
{
"created_at": "2026-03-01T05:06:04.880Z",
"description": "IXP in Australia",
"group_id": "0h50tbzgz6n8",
"id": "p823pmtmf6ts",
"name": "IXP",
"page_id": "rtthtlr99830",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-22T21:02:48.261Z"
}
],
"created_at": "2026-10-22T12:59:13.000Z",
"id": "qwbcltvzqxpt",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "p823pmtmf6ts",
"name": "Australia - IXP",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "Issue is resolved from Google.",
"created_at": "2026-10-22T21:02:48.311Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-22T21:02:48.311Z",
"id": "hnqnw3cs0sc1",
"incident_id": "qwbcltvzqxpt",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T21:02:48.311Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p823pmtmf6ts",
"name": "Australia - IXP",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We are observing improvements on Gemini 2.5 Flash traffic. We are monitoring a possible fix from Google.",
"created_at": "2026-10-22T17:56:41.599Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-22T17:56:41.599Z",
"id": "802knpvh607x",
"incident_id": "qwbcltvzqxpt",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T17:56:41.599Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p823pmtmf6ts",
"name": "Australia - IXP",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We've contacted Google Support, and they've started investigating on their end. We'll keep monitoring and share updates as we have them.",
"created_at": "2026-10-22T15:42:01.081Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-22T15:42:01.081Z",
"id": "czls64k4bg5v",
"incident_id": "qwbcltvzqxpt",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T15:42:01.081Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p823pmtmf6ts",
"name": "Australia - IXP",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "Following initial recovery, the service has degraded again. We are investigating and working towards a resolution. We will provide further updates as they become available.",
"created_at": "2026-10-22T14:33:43.864Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T14:33:43.864Z",
"id": "mg9ltq5kfw80",
"incident_id": "qwbcltvzqxpt",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T14:33:43.864Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p823pmtmf6ts",
"name": "Australia - IXP",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "A fix has been deployed and the service is recovering. We are monitoring the system to ensure full recovery. Further updates will be shared soon",
"created_at": "2026-10-22T14:19:53.622Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-22T14:19:53.622Z",
"id": "2vgygbtys32v",
"incident_id": "qwbcltvzqxpt",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T14:19:53.622Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "p823pmtmf6ts",
"name": "Australia - IXP",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We have identified an issue affecting IXP for customers hosted in the Australia region. Since 10:24 UTC, requests that use the Gemini 2.5 Flash model are failing in the region. Other models are not affected. Customer data continues to be processed only within Australia. \nWe are working with Google to restore service",
"created_at": "2026-10-22T12:59:13.801Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-22T12:59:13.000Z",
"id": "vvt61l0b5z6t",
"incident_id": "qwbcltvzqxpt",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-22T14:15:57.081Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-22T14:19:53.622Z",
"name": "Gemini-2.5-Flash not working in Australia",
"page_id": "rtthtlr99830",
"postmortem_body": "## Customer impact\n\nBetween October 22, 2026 at 10:24 am UTC and October 22, 2026 at 6:04 pm UTC, document processing requests that used Gemini 2.5 Flash in the Australia region failed with capacity errors. The impact lasted approximately 7 hours and 40 minutes.\n\nA subset of customers using document extraction workflows in that region were affected. During a temporary in-region fallback attempt, some extraction requests recovered briefly, but word-level attribution and generated page text were degraded until the fallback also began failing.\n\n## Root cause\n\nThe root cause was a provider-side regional network disruption during concurrent maintenance. This caused severe bandwidth throttling in the provider's model-serving environment, which prevented the backend systems from loading the model data needed to serve Gemini 2.5 Flash and Gemini 3.5 Flash requests reliably in the affected region.\n\n## Detection\n\nThe issue was detected through automated alerting that identified rate-limiting errors on Gemini 2.5 Flash in the Australia region. Our engineering team confirmed through telemetry that affected requests were failing consistently and began coordinating a response.\n\n## Response\n\nAt 1:40 pm UTC, we routed affected Gemini 2.5 Flash traffic to Gemini 3.5 Flash within the same region to preserve data residency. Initial telemetry showed no capacity errors after that change, but by 2:10 pm UTC, Gemini 3.5 Flash was also returning capacity errors, so the fallback was no longer viable.\n\nAt 2:05 pm UTC, we adjusted related document processing paths so attribution and page-text generation would use the same in-region fallback; that adjustment completed at approximately 2:09 pm UTC. After the fallback also failed, we reverted the routing change, and by 4:28 pm UTC traffic was back on the expected Gemini 2.5 Flash path.\n\nAt 5:38 pm UTC, telemetry showed improving Gemini 2.5 Flash success rates. From 6:04 pm UTC onward, no further capacity errors were observed, and subsequent requests succeeded.\n\n## Follow up\n\n1. Complete the reserved model capacity assessment for the affected region using the observed traffic patterns\n2. Evaluate retry and backoff behavior in document processing clients to reduce retry amplification during provider-side capacity failures.",
"postmortem_body_last_updated_at": "2026-10-26T03:20:24.659Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-26T03:20:24.659Z",
"reminder_intervals": null,
"resolved_at": "2026-10-22T21:02:48.311Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/cr6m5z76mzzp",
"started_at": "2026-10-22T12:59:13.000Z",
"status": "resolved",
"updated_at": "2026-10-26T03:20:24.766Z"
},
{
"components": [
{
"created_at": "2026-07-07T01:57:59.310Z",
"description": null,
"group_id": "2r9tpytl7r6r",
"id": "s5fqcv19k46r",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 3,
"showcase": true,
"start_date": "2026-07-07T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.117Z"
},
{
"created_at": "2026-03-10T03:56:21.005Z",
"description": null,
"group_id": "m1b1ynftp3sv",
"id": "1wf1xmhrhq7s",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 15,
"showcase": true,
"start_date": "2026-03-10T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.143Z"
},
{
"created_at": "2026-07-07T02:02:11.201Z",
"description": null,
"group_id": "2r9tpytl7r6r",
"id": "41nzjqkhlf1r",
"name": "Solutions Management",
"page_id": "rtthtlr99830",
"position": 19,
"showcase": true,
"start_date": "2026-07-07T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.166Z"
},
{
"created_at": "2026-03-01T05:02:49.712Z",
"description": "Solutions Management in United States",
"group_id": "8md89tvvp0jc",
"id": "z8351chcxsms",
"name": "Solutions Management",
"page_id": "rtthtlr99830",
"position": 20,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.189Z"
},
{
"created_at": "2026-03-01T05:03:39.216Z",
"description": "Solutions Management in Japan",
"group_id": "q9rqbbwgxqfz",
"id": "vm9nsmnz3dz8",
"name": "Solutions Management",
"page_id": "rtthtlr99830",
"position": 20,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.211Z"
},
{
"created_at": "2026-03-01T05:06:07.861Z",
"description": "Solutions Management in Australia",
"group_id": "0h50tbzgz6n8",
"id": "6j0bgjbv7hwv",
"name": "Solutions Management",
"page_id": "rtthtlr99830",
"position": 20,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.233Z"
},
{
"created_at": "2026-03-01T05:06:56.696Z",
"description": "Solutions Management in Canada",
"group_id": "3w1cgj3h4gyt",
"id": "4jkmxs1hv7zt",
"name": "Solutions Management",
"page_id": "rtthtlr99830",
"position": 20,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.255Z"
},
{
"created_at": "2026-03-01T05:07:49.072Z",
"description": "Solutions Management in UK",
"group_id": "sfkr8z0m2j3k",
"id": "lgjh5ml6hbqp",
"name": "Solutions Management",
"page_id": "rtthtlr99830",
"position": 20,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.276Z"
},
{
"created_at": "2024-03-01T13:28:48.537Z",
"description": null,
"group_id": "bt4fyv3z54lv",
"id": "5txf886jydqg",
"name": "Solutions Management",
"page_id": "rtthtlr99830",
"position": 22,
"showcase": true,
"start_date": "2024-02-29T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.298Z"
},
{
"created_at": "2025-10-30T09:22:39.428Z",
"description": null,
"group_id": "bt4fyv3z54lv",
"id": "xz3mpdh89145",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 26,
"showcase": true,
"start_date": "2025-08-02T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.320Z"
},
{
"created_at": "2026-03-01T05:03:01.700Z",
"description": "Agents in United States",
"group_id": "8md89tvvp0jc",
"id": "2h3vc1w1lkyn",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.341Z"
},
{
"created_at": "2026-03-01T05:03:51.323Z",
"description": "Agents in Japan",
"group_id": "q9rqbbwgxqfz",
"id": "jvls0hs1h4vq",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.511Z"
},
{
"created_at": "2026-03-01T05:04:41.054Z",
"description": "Agents in India",
"group_id": "k33cxmfzs4sq",
"id": "rqnswks3x9fk",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.533Z"
},
{
"created_at": "2026-03-01T05:05:31.865Z",
"description": "Agents in Singapore",
"group_id": "1bk483dn7k09",
"id": "dspkt4m2p068",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.555Z"
},
{
"created_at": "2026-03-01T05:06:20.072Z",
"description": "Agents in Australia",
"group_id": "0h50tbzgz6n8",
"id": "2n16xr60pkhs",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.576Z"
},
{
"created_at": "2026-03-01T05:07:08.724Z",
"description": "Agents in Canada",
"group_id": "3w1cgj3h4gyt",
"id": "w0l5dmd6nqb4",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.597Z"
},
{
"created_at": "2026-03-01T05:08:01.017Z",
"description": "Agents in UK",
"group_id": "sfkr8z0m2j3k",
"id": "x643741pxvfv",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.619Z"
},
{
"created_at": "2026-03-01T05:09:42.278Z",
"description": "Agents in Switzerland",
"group_id": "qt4tq8wc76h0",
"id": "tkb8f3wcwxy5",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.641Z"
},
{
"created_at": "2026-03-01T05:10:30.234Z",
"description": "Agents in Delayed US",
"group_id": "hdlv8jj8xx7k",
"id": "8zj0q2h51fwk",
"name": "Agents",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-15T13:04:18.663Z"
}
],
"created_at": "2026-10-15T12:27:19.665Z",
"id": "spghbq872ty9",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "5txf886jydqg",
"name": "European Union - Solutions Management",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "xz3mpdh89145",
"name": "European Union - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "z8351chcxsms",
"name": "United States - Solutions Management",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "2h3vc1w1lkyn",
"name": "United States - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "vm9nsmnz3dz8",
"name": "Japan - Solutions Management",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "jvls0hs1h4vq",
"name": "Japan - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "rqnswks3x9fk",
"name": "India - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "dspkt4m2p068",
"name": "Singapore - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "6j0bgjbv7hwv",
"name": "Australia - Solutions Management",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "2n16xr60pkhs",
"name": "Australia - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4jkmxs1hv7zt",
"name": "Canada - Solutions Management",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "w0l5dmd6nqb4",
"name": "Canada - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "lgjh5ml6hbqp",
"name": "UK - Solutions Management",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "x643741pxvfv",
"name": "UK - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tkb8f3wcwxy5",
"name": "Switzerland - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "8zj0q2h51fwk",
"name": "Delayed US - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "1wf1xmhrhq7s",
"name": "UAE - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "s5fqcv19k46r",
"name": "South Korea - Agents",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "41nzjqkhlf1r",
"name": "South Korea - Solutions Management",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved. Between 11:15 and 12:45 UTC on October 15, an internal configuration issue prevented some events from being delivered between UiPath services across multiple regions. The issue was resolved by rolling back the change, and new activity is being processed normally. Some work initiated or in progress during this window may require attention:\n\n- Agents: Agents waiting on Action Center tasks or automation tool results may remain stuck. Jobs started from Orchestrator may also appear stuck or time out.\n- Maestro: Processes waiting on user tasks may not have resumed after task completion.\n- Apps: Queue item statuses may be outdated, and actions triggered by queue item completion may not have run.\n- Autopilot for Everyone: Conversations waiting for automation results may remain stalled.\n- Document Understanding and Context Grounding: Automations and agents waiting for extraction, validation, or indexing may remain suspended.\n- Integration Service: Connector events may not have triggered the expected processes or notifications.\n- Solutions and Studio Web: Solution installs, uninstalls, publishes, deployments, and validations initiated during this window may have failed or timed out. Retrying these operations may resolve the issue.\n\nWe apologize for the disruption and thank you for your patience.",
"created_at": "2026-10-15T13:22:26.241Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T13:22:26.000Z",
"id": "vb1bngkv62yv",
"incident_id": "spghbq872ty9",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T15:05:50.655Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "5txf886jydqg",
"name": "European Union - Solutions Management",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "xz3mpdh89145",
"name": "European Union - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "z8351chcxsms",
"name": "United States - Solutions Management",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "2h3vc1w1lkyn",
"name": "United States - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "vm9nsmnz3dz8",
"name": "Japan - Solutions Management",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "jvls0hs1h4vq",
"name": "Japan - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "rqnswks3x9fk",
"name": "India - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "dspkt4m2p068",
"name": "Singapore - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "6j0bgjbv7hwv",
"name": "Australia - Solutions Management",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "2n16xr60pkhs",
"name": "Australia - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "4jkmxs1hv7zt",
"name": "Canada - Solutions Management",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "w0l5dmd6nqb4",
"name": "Canada - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "lgjh5ml6hbqp",
"name": "UK - Solutions Management",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "x643741pxvfv",
"name": "UK - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "tkb8f3wcwxy5",
"name": "Switzerland - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "8zj0q2h51fwk",
"name": "Delayed US - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "1wf1xmhrhq7s",
"name": "UAE - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "s5fqcv19k46r",
"name": "South Korea - Agents",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "41nzjqkhlf1r",
"name": "South Korea - Solutions Management",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "The issue has been mitigated. We are monitoring the affected services across regions to ensure stability and confirm that job starts and solution deployments are functioning normally.",
"created_at": "2026-10-15T13:04:18.714Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T13:04:18.714Z",
"id": "t3vl6lhrtm9g",
"incident_id": "spghbq872ty9",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T13:04:18.714Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "vm9nsmnz3dz8",
"name": "Japan - Solutions Management",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "jvls0hs1h4vq",
"name": "Japan - Agents",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "rqnswks3x9fk",
"name": "India - Agents",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "2n16xr60pkhs",
"name": "Australia - Agents",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "41nzjqkhlf1r",
"name": "South Korea - Solutions Management",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "Since approximately 11:17 UTC, customers across multiple regions have experienced issues with UiPath Agents and Solutions Management. New jobs may remain in Pending status, and running agents may experience delays when processing escalations or waiting for tool results.\nWe are investigating the issue and working toward a resolution.",
"created_at": "2026-10-15T12:27:19.936Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T12:27:19.936Z",
"id": "lxt64z7mrx9f",
"incident_id": "spghbq872ty9",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T12:27:19.936Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-15T13:04:18.714Z",
"name": "Multiple Regions - Multiple Services - Delayed and missing event notifications across several products",
"page_id": "rtthtlr99830",
"postmortem_body": "**Customer impact**\n\nOn October 15, 2026, from 11:15 to 12:50 UTC (about 1 hour and 35 minutes), some events exchanged between UiPath cloud services were not delivered. Customers in all commercial cloud regions were affected. During this window:\n\n- New agent runs could stay in Pending status. Running agents waiting on a tool result, a human escalation, document extraction, or context grounding might not resume.\n- Conversational agents and Autopilot for Everyone conversations that started an automation might not have received its result.\n- Maestro process instances waiting on a user task might not have moved forward after the task was completed, and processes triggered by Integration Service connector events might not have started.\n- Apps showing or reacting to queue item status could show outdated information.\n- Solution installs, uninstalls, publishes, and deployments could fail or time out.\n- Some user, group, and folder changes made in the window might not have propagated to all services.\n\nWork started after 12:50 UTC was not affected. Robot job execution, queues, and triggers handled inside Orchestrator kept working normally.\n\n**Root cause**\n\nWe were moving the delivery of internal service configuration to a new mechanism, a change intended to have no effect on customers. When this change was applied to the event delivery service, the configuration it produced was missing a required value. Without it, the service treated several product integrations as turned off and silently stopped delivering their events. Because the change reached all production regions at the same time, every region was affected.\n\n**Detection**\n\nThe issue was detected at 11:51 UTC, about 35 minutes after impact began, when monitoring alerted on agent jobs remaining in Pending status. Reports of solution deployments timing out followed, and an incident was declared at 12:12 UTC.\n\n**Response**\n\nFollowing the first detection at 11:51 UTC, the configuration change was identified and a rollback was started at 12:00 UTC. An initial rollback at 12:14 UTC did not fully restore delivery. The team then stopped the new configuration delivery process and restored the previous configuration. Event delivery began recovering at 12:40 UTC, and full recovery across all regions was confirmed at 12:50 UTC. Where possible, we replayed missed events so that affected work could continue.\n\n**Follow up**\n\n1. The new configuration delivery process has been paused for the affected service and will stay paused until the configuration is generated correctly.\n2. Validate configuration produced by the new delivery mechanism for required values before it is applied to any environment.\n3. Roll out changes to configuration delivery progressively across regions instead of to all regions at once.\n4. Improve our monitoring to detect within minutes when events stop being delivered to a product integration that normally receives them.\n5. Improve our ability to recover and redeliver events missed during a disruption.",
"postmortem_body_last_updated_at": "2026-10-21T06:47:10.023Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-21T06:47:10.023Z",
"reminder_intervals": null,
"resolved_at": "2026-10-15T13:22:26.000Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/hqt6c7pc8bzg",
"started_at": "2026-10-15T12:27:19.636Z",
"status": "resolved",
"updated_at": "2026-10-21T06:47:10.135Z"
},
{
"components": [
{
"created_at": "2026-03-01T05:02:17.880Z",
"description": "Orchestrator in United States",
"group_id": "8md89tvvp0jc",
"id": "8c2g6j81glkx",
"name": "Orchestrator",
"page_id": "rtthtlr99830",
"position": 2,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-07T19:06:26.394Z"
}
],
"created_at": "2026-10-15T12:03:20.614Z",
"id": "w70d8q71bbdf",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "8c2g6j81glkx",
"name": "United States - Orchestrator",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "Some customers in the US region experienced delays in robot log ingestion, with increased activity observed between approximately 11:20 and 11:45 UTC. The issue has been resolved, and log ingestion has returned to normal. Thank you for your patience.",
"created_at": "2026-10-15T12:03:20.683Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-15T12:03:20.683Z",
"id": "mlhs57626h3z",
"incident_id": "w70d8q71bbdf",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-15T12:03:20.683Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "US – Orchestrator - Delays in Robot Log Ingestion",
"page_id": "rtthtlr99830",
"postmortem_body": "## Customer impact\n\nBetween approximately October 15, 2026 at 11:20 pm UTC and approximately October 15, 2026 at 11:45 pm UTC, some customers in the US East region experienced delays in robot log ingestion in UiPath Orchestrator. Robot logs were not up to date during the affected period.\n\nThe impact lasted approximately 20 minutes.\n\n## Root cause\n\nDuring the incident, 2 of the 13 service instances handling robot log ingestion were observed failing to connect to the event ingestion service. Only requests to the event ingestion service were failing, and only on those two instances. This caused some robot log ingestion requests to not complete successfully, resulting in delayed log data for affected customers.\n\nThe root cause of why only those two instances lost connectivity to the event ingestion service was not determined during the incident. The team committed to a post-incident review of logs from the affected instances to identify the underlying cause. We believe it to be an unreported infrastructure issue from our cloud provider.\n\n## Detection\n\nThe issue was detected through an automated alert at approximately 11:33 pm UTC on October 15, 2026, and confirmed through telemetry showing exceptions from the two affected service instances.\n\n## Response\n\nAfter detection, telemetry confirmed that only two service instances were generating exceptions. Responders removed those two unhealthy instances from service, and replacement instances started automatically.\n\nWithin approximately 20 minutes of the first alert — by approximately 11:53 pm UTC — telemetry indicated recovery and synthetic monitoring checks were passing. A broader restart of all service instances was considered but not performed, as the team determined it should not proceed without first understanding the root cause.\n\n## Follow up\n\n1. A post-incident review of logs from the affected service instances revealed the issue was infrastructure related, and we are following up with out cloud infrastructure provider for more details.\n2. An update will be posted if the investigation yields further findings.",
"postmortem_body_last_updated_at": "2026-10-22T11:10:23.829Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-22T10:13:04.817Z",
"reminder_intervals": null,
"resolved_at": "2026-10-15T12:03:20.683Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/fxykbtbqk6p0",
"started_at": "2026-10-15T12:03:20.607Z",
"status": "resolved",
"updated_at": "2026-10-22T11:10:23.831Z"
},
{
"components": [],
"created_at": "2026-10-14T07:26:42.660Z",
"id": "y92n3217kkxp",
"impact": "none",
"impact_override": "none",
"incident_updates": [
{
"affected_components": null,
"body": "Starting with 6:50 UTC we have experienced a global outage that has affected the loading of apps and coded apps all regions except for India and UK. Outage has been resolved and the services are fully operational.\nImpact: No ongoing user impact.",
"created_at": "2026-10-14T07:43:27.696Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-14T07:43:27.696Z",
"id": "mfrjpqd3nmj8",
"incident_id": "y92n3217kkxp",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-14T07:43:27.696Z",
"wants_twitter_update": false
},
{
"affected_components": null,
"body": "Starting with 6:50 UTC we have experienced a global outage that has affected the loading of apps and coded apps all regions except for India and UK. Outage has been resolved and the services are fully operational.\nImpact: No ongoing user impact.",
"created_at": "2026-10-14T07:26:42.724Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-14T07:26:42.724Z",
"id": "mtwdcsjn9l5z",
"incident_id": "y92n3217kkxp",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-14T07:26:42.724Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-14T07:26:42.724Z",
"name": "Failures in App and Coded Apps requests in all regions",
"page_id": "rtthtlr99830",
"postmortem_body": "## Customer impact\n\nBetween 06:38 UTC and 07:21 UTC on October 14, 2026, a portion of requests to UiPath Apps, Coded Apps, and solutions that depend on them failed in eleven regions: Europe, United States, Community, Japan, Singapore, Australia, Canada, South Korea, Switzerland, UAE, and the delayed US environment. The India and United Kingdom regions were not affected.\n\nAbout 2 percent of requests failed overall, between 0.6 and 5 percent depending on the region. Affected users saw applications fail to load, requests return server errors, or slow responses. In regions where service pods had to be restarted, Apps was unreachable for up to about eight minutes. Response times were back to normal in every region by 07:40 UTC.\n\nMost users of already running published apps noticed little or nothing. App definitions are cached in the browser for up to 24 hours, app data comes from Data Service and Orchestrator rather than this database, and the service kept using the database connections it already held. The requests that failed were those needing a new database connection: opening the designer or the apps list, loading an app not yet cached, downloading documents and images stored in Apps, and saving or publishing.\n\n## Root cause\n\nAt 06:38 UTC, while database access was being set up for a new regional deployment, the password of the database user that the Apps service uses was changed. That user is shared by the production database clusters of all affected regions, so the change applied to all of them at once. The running services still held the previous password and could no longer open new database connections. The India and United Kingdom regions do not use this credential and were not affected.\n\nThe change was meant to create a separate database user for the new region. Instead, the password of the existing user that all regions share was changed. Our change process for database credentials did not clearly separate creating a new credential from modifying an existing one, and it did not require a check of how many clusters and regions a credential change would affect. Combined with a single database user being shared across regions, this allowed a change intended for one region to cause a multi-region outage.\n\n## Detection\n\nOur internal monitoring raised the first alert at 06:41 UTC, three minutes after the first failures, and alerts for every affected region followed by 06:46 UTC. At 06:50 UTC the on-call engineer confirmed that every production cluster was being refused by its database and started the incident process.\n\n## Response\n\nAt 07:05 UTC the previous password was restored and authentication failures stopped everywhere. Most regions reconnected automatically. In the regions that did not, the service pods were restarted or redeployed: Delayed US at 07:21 UTC and Switzerland by 07:30 UTC. Request failures stopped at 07:21 UTC, response times were back to baseline by 07:40 UTC, and we continued monitoring until the incident was resolved at 07:43 UTC.\n\n## Follow up\n\nTo prevent this from happening again, we are:\n\n- Introducing a defined procedure for creating and rotating database credentials, with a pre-change review that states which clusters and regions share the credential, and a staged rollout that is verified in one region before it is applied more widely.\n- Moving to separate, least-privilege database users for each region, so that a change in one region cannot affect another.\n- Automatically declaring an incident and paging the database owners when the same database error is detected in several regions at once, instead of relying on manual escalation from individual regional alerts.\n- Enabling services to pick up updated database credentials without a restart, and to reconnect faster after an authentication failure.",
"postmortem_body_last_updated_at": "2026-10-23T12:08:40.575Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-23T12:08:40.575Z",
"reminder_intervals": null,
"resolved_at": "2026-10-14T07:43:27.696Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/6kswhyv0b1cf",
"started_at": "2026-10-14T07:26:42.653Z",
"status": "resolved",
"updated_at": "2026-10-23T12:08:40.807Z"
},
{
"components": [
{
"created_at": "2021-09-27T15:02:40.762Z",
"description": null,
"group_id": "bt4fyv3z54lv",
"id": "ncrpjwb1dl2j",
"name": "Computer Vision",
"page_id": "rtthtlr99830",
"position": 8,
"showcase": true,
"start_date": "2021-09-27T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.346Z"
},
{
"created_at": "2026-03-01T05:02:29.636Z",
"description": "Computer Vision in United States",
"group_id": "8md89tvvp0jc",
"id": "846j2gs041rk",
"name": "Computer Vision",
"page_id": "rtthtlr99830",
"position": 8,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.378Z"
},
{
"created_at": "2021-07-09T12:54:15.424Z",
"description": null,
"group_id": "bt4fyv3z54lv",
"id": "brzjkmc9ccky",
"name": "Document Understanding",
"page_id": "rtthtlr99830",
"position": 12,
"showcase": true,
"start_date": "2021-07-09T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.405Z"
},
{
"created_at": "2026-03-01T05:02:37.143Z",
"description": "Document Understanding in United States",
"group_id": "8md89tvvp0jc",
"id": "4knfzrmmkxcm",
"name": "Document Understanding",
"page_id": "rtthtlr99830",
"position": 12,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.437Z"
},
{
"created_at": "2026-03-01T05:03:26.454Z",
"description": "Document Understanding in Japan",
"group_id": "q9rqbbwgxqfz",
"id": "tr7rvtdbkx2j",
"name": "Document Understanding",
"page_id": "rtthtlr99830",
"position": 12,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.462Z"
},
{
"created_at": "2026-03-01T05:04:16.320Z",
"description": "Document Understanding in India",
"group_id": "k33cxmfzs4sq",
"id": "fdbs58d68mzf",
"name": "Document Understanding",
"page_id": "rtthtlr99830",
"position": 12,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:26.627Z"
},
{
"created_at": "2026-03-01T05:05:05.628Z",
"description": "Document Understanding in Singapore",
"group_id": "1bk483dn7k09",
"id": "fbqmb4cj5l8s",
"name": "Document Understanding",
"page_id": "rtthtlr99830",
"position": 12,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.485Z"
},
{
"created_at": "2026-03-01T05:05:55.905Z",
"description": "Document Understanding in Australia",
"group_id": "0h50tbzgz6n8",
"id": "sfj6hl63qgyv",
"name": "Document Understanding",
"page_id": "rtthtlr99830",
"position": 12,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.508Z"
},
{
"created_at": "2026-03-01T05:06:44.706Z",
"description": "Document Understanding in Canada",
"group_id": "3w1cgj3h4gyt",
"id": "mwyq71qp147r",
"name": "Document Understanding",
"page_id": "rtthtlr99830",
"position": 12,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.539Z"
},
{
"created_at": "2026-03-01T05:10:06.111Z",
"description": "Document Understanding in Delayed US",
"group_id": "hdlv8jj8xx7k",
"id": "8hhbgkkkhv00",
"name": "Document Understanding",
"page_id": "rtthtlr99830",
"position": 12,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.568Z"
},
{
"created_at": "2026-03-01T05:02:46.597Z",
"description": "IXP in United States",
"group_id": "8md89tvvp0jc",
"id": "j6p5vv4572jc",
"name": "IXP",
"page_id": "rtthtlr99830",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.600Z"
},
{
"created_at": "2026-03-01T05:03:35.756Z",
"description": "IXP in Japan",
"group_id": "q9rqbbwgxqfz",
"id": "2xgkcw3k95j8",
"name": "IXP",
"page_id": "rtthtlr99830",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T14:42:26.633Z"
},
{
"created_at": "2026-03-01T05:05:14.786Z",
"description": "IXP in Singapore",
"group_id": "1bk483dn7k09",
"id": "kyrf6l3czdwt",
"name": "IXP",
"page_id": "rtthtlr99830",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:26.651Z"
},
{
"created_at": "2026-03-01T05:06:04.880Z",
"description": "IXP in Australia",
"group_id": "0h50tbzgz6n8",
"id": "p823pmtmf6ts",
"name": "IXP",
"page_id": "rtthtlr99830",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-22T21:02:48.261Z"
},
{
"created_at": "2026-03-01T05:06:53.694Z",
"description": "IXP in Canada",
"group_id": "3w1cgj3h4gyt",
"id": "sts7j63f9yrg",
"name": "IXP",
"page_id": "rtthtlr99830",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:26.695Z"
},
{
"created_at": "2026-03-01T05:10:15.210Z",
"description": "IXP in Delayed US",
"group_id": "hdlv8jj8xx7k",
"id": "yswh8f5ybxl5",
"name": "IXP",
"page_id": "rtthtlr99830",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:26.722Z"
},
{
"created_at": "2026-03-01T05:11:04.647Z",
"description": "IXP in Delayed EU",
"group_id": "xqvsw9dkctvb",
"id": "4547xj65sv7h",
"name": "IXP",
"page_id": "rtthtlr99830",
"position": 18,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:26.751Z"
},
{
"created_at": "2022-11-15T15:57:17.953Z",
"description": null,
"group_id": "bt4fyv3z54lv",
"id": "ct2ycwn7t9v0",
"name": "IXP",
"page_id": "rtthtlr99830",
"position": 19,
"showcase": true,
"start_date": "2022-11-15T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:26.775Z"
},
{
"created_at": "2025-12-12T23:14:33.936Z",
"description": null,
"group_id": "bt4fyv3z54lv",
"id": "gcm3m5734d2m",
"name": "Autopilot for Developers",
"page_id": "rtthtlr99830",
"position": 28,
"showcase": true,
"start_date": "2025-12-12T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:26.803Z"
},
{
"created_at": "2026-03-01T05:03:03.245Z",
"description": "Autopilot for Developers in United States",
"group_id": "8md89tvvp0jc",
"id": "ljxlbf9bfsmm",
"name": "Autopilot for Developers",
"page_id": "rtthtlr99830",
"position": 29,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:26.969Z"
},
{
"created_at": "2026-03-01T05:05:33.386Z",
"description": "Autopilot for Developers in Singapore",
"group_id": "1bk483dn7k09",
"id": "9fqy9d8yw662",
"name": "Autopilot for Developers",
"page_id": "rtthtlr99830",
"position": 29,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:26.995Z"
},
{
"created_at": "2026-03-01T05:06:21.690Z",
"description": "Autopilot for Developers in Australia",
"group_id": "0h50tbzgz6n8",
"id": "ns9px4vr95jr",
"name": "Autopilot for Developers",
"page_id": "rtthtlr99830",
"position": 29,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:27.022Z"
},
{
"created_at": "2026-03-01T05:10:32.415Z",
"description": "Autopilot for Developers in Delayed US",
"group_id": "hdlv8jj8xx7k",
"id": "hzcphlfzgd5c",
"name": "Autopilot for Developers",
"page_id": "rtthtlr99830",
"position": 29,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T15:10:27.048Z"
}
],
"created_at": "2026-10-13T14:03:53.941Z",
"id": "0nxml8zd4r1d",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "ncrpjwb1dl2j",
"name": "European Union - Computer Vision",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "brzjkmc9ccky",
"name": "European Union - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ct2ycwn7t9v0",
"name": "European Union - IXP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "gcm3m5734d2m",
"name": "European Union - Autopilot for Developers",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "846j2gs041rk",
"name": "United States - Computer Vision",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4knfzrmmkxcm",
"name": "United States - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "j6p5vv4572jc",
"name": "United States - IXP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ljxlbf9bfsmm",
"name": "United States - Autopilot for Developers",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "tr7rvtdbkx2j",
"name": "Japan - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "2xgkcw3k95j8",
"name": "Japan - IXP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "fdbs58d68mzf",
"name": "India - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "fbqmb4cj5l8s",
"name": "Singapore - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kyrf6l3czdwt",
"name": "Singapore - IXP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "9fqy9d8yw662",
"name": "Singapore - Autopilot for Developers",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "sfj6hl63qgyv",
"name": "Australia - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "p823pmtmf6ts",
"name": "Australia - IXP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ns9px4vr95jr",
"name": "Australia - Autopilot for Developers",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "mwyq71qp147r",
"name": "Canada - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "sts7j63f9yrg",
"name": "Canada - IXP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "8hhbgkkkhv00",
"name": "Delayed US - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "yswh8f5ybxl5",
"name": "Delayed US - IXP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "hzcphlfzgd5c",
"name": "Delayed US - Autopilot for Developers",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4547xj65sv7h",
"name": "Delayed EU - IXP",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "This incident has been resolved following the renewal of the expired SSL certificate that affected several ML-dependent services, including IXP, Document Understanding, Computer Vision, Semantic Proxy and Autopilot for Developers. Service recovery has progressed across the affected regions, and automated alerts have been configured to help detect certificate expiry in advance and prevent similar disruptions in the future.\n\nThank you for your patience and understanding.",
"created_at": "2026-10-13T15:19:45.493Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T15:19:45.493Z",
"id": "lhxwsycc0888",
"incident_id": "0nxml8zd4r1d",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T15:19:45.493Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "ncrpjwb1dl2j",
"name": "European Union - Computer Vision",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "brzjkmc9ccky",
"name": "European Union - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ct2ycwn7t9v0",
"name": "European Union - IXP",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "gcm3m5734d2m",
"name": "European Union - Autopilot for Developers",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "846j2gs041rk",
"name": "United States - Computer Vision",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "4knfzrmmkxcm",
"name": "United States - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "j6p5vv4572jc",
"name": "United States - IXP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "ljxlbf9bfsmm",
"name": "United States - Autopilot for Developers",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "tr7rvtdbkx2j",
"name": "Japan - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "2xgkcw3k95j8",
"name": "Japan - IXP",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "fdbs58d68mzf",
"name": "India - Document Understanding",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "fbqmb4cj5l8s",
"name": "Singapore - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "kyrf6l3czdwt",
"name": "Singapore - IXP",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "9fqy9d8yw662",
"name": "Singapore - Autopilot for Developers",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "sfj6hl63qgyv",
"name": "Australia - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "p823pmtmf6ts",
"name": "Australia - IXP",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "ns9px4vr95jr",
"name": "Australia - Autopilot for Developers",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "mwyq71qp147r",
"name": "Canada - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "sts7j63f9yrg",
"name": "Canada - IXP",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "8hhbgkkkhv00",
"name": "Delayed US - Document Understanding",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "yswh8f5ybxl5",
"name": "Delayed US - IXP",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "hzcphlfzgd5c",
"name": "Delayed US - Autopilot for Developers",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "4547xj65sv7h",
"name": "Delayed EU - IXP",
"new_status": "operational",
"old_status": "major_outage"
}
],
"body": "A fix has been implemented and we are monitoring the results.",
"created_at": "2026-10-13T15:10:27.093Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T15:10:27.093Z",
"id": "hf77g7lkrykq",
"incident_id": "0nxml8zd4r1d",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T15:10:27.093Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "ncrpjwb1dl2j",
"name": "European Union - Computer Vision",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "brzjkmc9ccky",
"name": "European Union - Document Understanding",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "846j2gs041rk",
"name": "United States - Computer Vision",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "4knfzrmmkxcm",
"name": "United States - Document Understanding",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "j6p5vv4572jc",
"name": "United States - IXP",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "tr7rvtdbkx2j",
"name": "Japan - Document Understanding",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "2xgkcw3k95j8",
"name": "Japan - IXP",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "fbqmb4cj5l8s",
"name": "Singapore - Document Understanding",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "sfj6hl63qgyv",
"name": "Australia - Document Understanding",
"new_status": "operational",
"old_status": "major_outage"
},
{
"code": "mwyq71qp147r",
"name": "Canada - Document Understanding",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "8hhbgkkkhv00",
"name": "Delayed US - Document Understanding",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "A fix has been implemented and we are monitoring the results. The issue was limited to Document Understanding, Computer Vision, Semantic Proxy, Autopilot Studi, IXP. We will share another update once the services have fully recovered.",
"created_at": "2026-10-13T14:42:26.687Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T14:42:26.687Z",
"id": "wvs8v9j5y01f",
"incident_id": "0nxml8zd4r1d",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T14:42:26.687Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "ncrpjwb1dl2j",
"name": "European Union - Computer Vision",
"new_status": "major_outage",
"old_status": "partial_outage"
},
{
"code": "brzjkmc9ccky",
"name": "European Union - Document Understanding",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "ct2ycwn7t9v0",
"name": "European Union - IXP",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "gcm3m5734d2m",
"name": "European Union - Autopilot for Developers",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "846j2gs041rk",
"name": "United States - Computer Vision",
"new_status": "major_outage",
"old_status": "partial_outage"
},
{
"code": "4knfzrmmkxcm",
"name": "United States - Document Understanding",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "j6p5vv4572jc",
"name": "United States - IXP",
"new_status": "major_outage",
"old_status": "partial_outage"
},
{
"code": "ljxlbf9bfsmm",
"name": "United States - Autopilot for Developers",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "tr7rvtdbkx2j",
"name": "Japan - Document Understanding",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "2xgkcw3k95j8",
"name": "Japan - IXP",
"new_status": "major_outage",
"old_status": "partial_outage"
},
{
"code": "fdbs58d68mzf",
"name": "India - Document Understanding",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "st931vtzs3sr",
"name": "India - Autopilot for Developers",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "fbqmb4cj5l8s",
"name": "Singapore - Document Understanding",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "kyrf6l3czdwt",
"name": "Singapore - IXP",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "9fqy9d8yw662",
"name": "Singapore - Autopilot for Developers",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "sfj6hl63qgyv",
"name": "Australia - Document Understanding",
"new_status": "major_outage",
"old_status": "partial_outage"
},
{
"code": "p823pmtmf6ts",
"name": "Australia - IXP",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "ns9px4vr95jr",
"name": "Australia - Autopilot for Developers",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "mwyq71qp147r",
"name": "Canada - Document Understanding",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "sts7j63f9yrg",
"name": "Canada - IXP",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "8hhbgkkkhv00",
"name": "Delayed US - Document Understanding",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "yswh8f5ybxl5",
"name": "Delayed US - IXP",
"new_status": "major_outage",
"old_status": "major_outage"
},
{
"code": "hzcphlfzgd5c",
"name": "Delayed US - Autopilot for Developers",
"new_status": "major_outage",
"old_status": "operational"
},
{
"code": "4547xj65sv7h",
"name": "Delayed EU - IXP",
"new_status": "major_outage",
"old_status": "major_outage"
}
],
"body": "We are continuing to investigate this issue.",
"created_at": "2026-10-13T14:37:29.177Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T14:37:29.177Z",
"id": "7j9hn8q0cy3v",
"incident_id": "0nxml8zd4r1d",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T14:37:29.177Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "ncrpjwb1dl2j",
"name": "European Union - Computer Vision",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "brzjkmc9ccky",
"name": "European Union - Document Understanding",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "846j2gs041rk",
"name": "United States - Computer Vision",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "4knfzrmmkxcm",
"name": "United States - Document Understanding",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "j6p5vv4572jc",
"name": "United States - IXP",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "tr7rvtdbkx2j",
"name": "Japan - Document Understanding",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "2xgkcw3k95j8",
"name": "Japan - IXP",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "fbqmb4cj5l8s",
"name": "Singapore - Document Understanding",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "sfj6hl63qgyv",
"name": "Australia - Document Understanding",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "mwyq71qp147r",
"name": "Canada - Document Understanding",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "8hhbgkkkhv00",
"name": "Delayed US - Document Understanding",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are currently investigating an outage affecting UiPath Cloud services across multiple regions. Users are unable to access services hosted in the affected regions. Next update: Additional information will be shared as it becomes available.",
"created_at": "2026-10-13T14:03:54.539Z",
"custom_tweet": null,
"deliver_notifications": false,
"display_at": "2026-10-13T14:03:54.539Z",
"id": "7zw4y03zm16d",
"incident_id": "0nxml8zd4r1d",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T14:15:21.032Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "Multiple Regions - Multiple Services Failing Due to Certificate Expiry",
"page_id": "rtthtlr99830",
"postmortem_body": "## Customer Impact\n\nBetween 13:37 and 14:40 UTC on October 13, 2026, customers across multiple regions were unable to use IXP Unstructured Documents, IXP Communications Mining and Document Understanding. Computer Vision and Autopilot for Developers features that depend on the same machine learning services also failed.\n\nAffected customers saw requests fail with \"500 Internal Server Error\" responses or time out, IXP projects that would not open, and Computer Vision activities failing in running automations. Service was disrupted for approximately 1 hour. No customer data was lost.\n\n## Root Cause\n\nThe incident was caused by an expired security certificate used to establish secure connections to these services. The certificate is renewed by an automated process. That process was no longer executing successfully and the failures did not raise an alert, so the certificate was not replaced before it expired. Secure connections to the affected services were rejected from 13:37 UTC.\n\n## Detection\n\nAutomated monitoring began alerting on errors and missing traffic at 13:40 UTC, and the incident was declared at 13:54 UTC. The expired certificate was identified at 14:00 UTC, and the first status page update was posted at 14:03 UTC.\n\nNo alert warned that the certificate was approaching expiry, and the repeated failures of the automated renewal were not reported to the owning team, so the problem was only detected once customers were affected.\n\n## Response\n\nAt 14:06 UTC the team confirmed that the certificate had expired because the automated renewal had been failing. At 14:18 UTC a new credential was issued for the renewal process. The renewal was started manually at 14:19 UTC and completed at 14:33 UTC, and by 14:37 UTC every customer-facing endpoint had been verified to serve the renewed certificate. Traffic returned to normal levels by approximately 14:40 UTC. After a period of monitoring, the incident was resolved at 15:19 UTC.\n\n## Follow-Up\n\nTo help prevent similar incidents in the future, we are making the following improvements:\n\n- Alert well in advance of any of these certificates approaching expiry.\n- Notify the owning team whenever the automated renewal fails.\n- Add checks that alert as soon as secure connections to these services fail.\n- Strengthen ownership of the certificate renewal process and the credentials it relies on.",
"postmortem_body_last_updated_at": "2026-10-26T03:17:26.789Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-26T03:17:26.789Z",
"reminder_intervals": null,
"resolved_at": "2026-10-13T15:19:45.493Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/f31bzg6qclk9",
"started_at": "2026-10-13T14:03:53.905Z",
"status": "resolved",
"updated_at": "2026-10-26T03:17:26.934Z"
},
{
"components": [
{
"created_at": "2026-03-01T05:03:11.151Z",
"description": "Orchestrator in Japan",
"group_id": "q9rqbbwgxqfz",
"id": "cpvmj871ptxl",
"name": "Orchestrator",
"page_id": "rtthtlr99830",
"position": 2,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T10:57:35.803Z"
}
],
"created_at": "2026-10-13T10:32:04.270Z",
"id": "mwbgvfsstj75",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "cpvmj871ptxl",
"name": "Japan - Orchestrator",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "This issue has been resolved. Robot execution logs are syncing and are visible in Orchestrator again for Japan Orchestrator. Thank you for your patience.",
"created_at": "2026-10-13T10:57:35.861Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T10:57:35.861Z",
"id": "gdv1krwdfxht",
"incident_id": "mwbgvfsstj75",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T10:57:35.861Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "cpvmj871ptxl",
"name": "Japan - Orchestrator",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating issue where robot execution logs are not showing up in Orchestrator. we are working to determine the full scope and root cause.",
"created_at": "2026-10-13T10:32:04.370Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-13T10:32:04.370Z",
"id": "smyg2v57lgrd",
"incident_id": "mwbgvfsstj75",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-13T10:32:04.370Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": null,
"name": "JAPAN - Orchestrator - Robot logs are not showing up for some Customers",
"page_id": "rtthtlr99830",
"postmortem_body": "## Customer impact\n\nBetween approximately 07:00 UTC on 9 Oct 2026 and 10:45 UTC on 13 Oct 2026, about 6% of robot log entries for customers in the Japan region did not appear in Orchestrator. Job execution and all other Orchestrator functionality were not affected. Missing entries became visible in Orchestrator after the issue was fixed.\n\n## Root cause\n\nRobot logs are placed on a queue split into 32 partitions, and an ingestion service reads each partition and stores the logs for Orchestrator. On October 9, the ingestion service in the Japan region stopped reading from 2 of the 32 partitions without reporting an error. Logs on those partitions were kept in the queue but not stored, so they did not appear in Orchestrator. Our monitoring checked the service as a whole, not each partition, so the issue was not detected automatically.\n\n## Detection\n\nThe issue was detected through customer reports. We declared an incident at 10:13 UTC on October 13 and posted a status page notification at 10:32 UTC.\n\n## Response\n\nWe restarted the ingestion service at approximately 10:41 UTC, and it resumed reading all partitions. The logs held in the queue were then processed and became visible in Orchestrator. The incident was resolved at 10:57 UTC.\n\n## Follow up\n\n* We are adding monitoring that alerts us when any single partition stops being read, so this kind of issue is detected within minutes.",
"postmortem_body_last_updated_at": "2026-10-22T13:28:43.679Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-22T13:25:33.511Z",
"reminder_intervals": null,
"resolved_at": "2026-10-13T10:57:35.861Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/8dkj8py238m3",
"started_at": "2026-10-13T10:32:04.263Z",
"status": "resolved",
"updated_at": "2026-10-22T13:28:43.681Z"
},
{
"components": [
{
"created_at": "2026-03-01T05:02:57.147Z",
"description": "Agentic Orchestration in United States",
"group_id": "8md89tvvp0jc",
"id": "3dxdt7wg53st",
"name": "Agentic Orchestration",
"page_id": "rtthtlr99830",
"position": 25,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-08T04:29:22.356Z"
}
],
"created_at": "2026-10-07T21:34:37.201Z",
"id": "fk7vmjm4s8rq",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "3dxdt7wg53st",
"name": "United States - Agentic Orchestration",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "The fix for the issue impacting HITL task creation for Maestro with HITL task SLA timers configured in US has been fully rolled out. HITL task creation has returned to normal.",
"created_at": "2026-10-08T04:29:22.398Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-08T04:29:22.398Z",
"id": "dm3nt4bmbxpn",
"incident_id": "fk7vmjm4s8rq",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-08T04:29:22.398Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3dxdt7wg53st",
"name": "United States - Agentic Orchestration",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We are rolling out fix for an outage impacting HITL task creation for Maestros with HITL task SLA timers configured in US and are working to mitigate the issue.\nImpact: Users may be unable to create HITL tasks within Maestros with HITL task SLA timers configured.\nNext update: Our teams are working to mitigate the issue and will share updates as available.",
"created_at": "2026-10-08T03:28:05.605Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-08T03:28:05.605Z",
"id": "tj57mf9g2jff",
"incident_id": "fk7vmjm4s8rq",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-08T03:28:05.605Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3dxdt7wg53st",
"name": "United States - Agentic Orchestration",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We are rolling out fix for an outage impacting HITL task creation for Maestros with HITL task SLA timers configured in US and are working to mitigate the issue.\nImpact: Users may be unable to create HITL tasks within Maestros with HITL task SLA timers configured.\nNext update: Our teams are working to mitigate the issue and will share updates as available.",
"created_at": "2026-10-08T02:29:54.649Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-08T02:29:54.649Z",
"id": "4v7kcl2x43p1",
"incident_id": "fk7vmjm4s8rq",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-08T02:29:54.649Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3dxdt7wg53st",
"name": "United States - Agentic Orchestration",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We have identified the cause for an outage impacting HITL task creation for Maestro in US and are working to mitigate the issue.\nImpact: Users may be unable to create HITL tasks.\nNext update: Our teams are working to mitigate the issue and will share updates as available.",
"created_at": "2026-10-08T01:30:43.384Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-08T01:30:43.384Z",
"id": "46fjh6yl8z6z",
"incident_id": "fk7vmjm4s8rq",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-08T01:30:43.384Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3dxdt7wg53st",
"name": "United States - Agentic Orchestration",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We have identified the cause for an outage impacting HITL task creation for Maestro in US and are working to mitigate the issue.\nImpact: Users may be unable to create HITL tasks.\nNext update: Our teams are working to mitigate the issue and will share updates as available.",
"created_at": "2026-10-08T00:29:02.013Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-08T00:29:02.013Z",
"id": "7r232z7jc2rs",
"incident_id": "fk7vmjm4s8rq",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-08T00:29:02.013Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3dxdt7wg53st",
"name": "United States - Agentic Orchestration",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We have identified the cause for an outage impacting HITL task creation for Maestro in US and are working to mitigate the issue.\nImpact: Users may be unable to create HITL tasks.\nNext update: Our teams are working to mitigate the issue and will share updates as available.",
"created_at": "2026-10-07T23:26:35.735Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-07T23:26:35.735Z",
"id": "dcrd46qkdhwt",
"incident_id": "fk7vmjm4s8rq",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-07T23:26:35.735Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3dxdt7wg53st",
"name": "United States - Agentic Orchestration",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We have identified the cause for an outage impacting HITL task creation for Maestro in US and are working to mitigate the issue.\nImpact: Users may be unable to create HITL tasks.\nNext update: Our teams are working to mitigate the issue and will share updates as available.",
"created_at": "2026-10-07T22:16:34.763Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-07T22:16:34.763Z",
"id": "bcdl1xd1bnk3",
"incident_id": "fk7vmjm4s8rq",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-07T22:16:34.763Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "3dxdt7wg53st",
"name": "United States - Agentic Orchestration",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are investigating reports of an outage impacting HITL task creation failures for Maestro in US.\nImpact: Users may be unable to create HITL tasks.\nNext update: Our teams are working to understand the cause and scope and will share updates as available.",
"created_at": "2026-10-07T21:34:37.451Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-07T21:34:37.451Z",
"id": "ywbwgxsphkt5",
"incident_id": "fk7vmjm4s8rq",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-07T21:34:37.451Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-08T02:29:54.649Z",
"name": "US - Maestro - HITL task creation failures within Maestros with HITL task SLA timers configured",
"page_id": "rtthtlr99830",
"postmortem_body": "## Customer impact\n\nOn 2026-10-05 between 9.30 PM UTC - 2026-10-07 3.30 AM UTC, Maestro BPMN and Case customers using HITL nodes with a configured Task Timer saw failures of their debug and production instances at the HITL stage, with 4xx . About 30 such instances across customers across US and Europe were identified, that were blocked with no workaround.\n\n## Root cause\n\nConfiguring a task SLA requires an object including the start and end criteria, the timer of the SLA, and the outcome on SLA breach. Customers configure that on the Maestro designer on the HITL node. The outcome was mistakenly kept non-mandatory on the design surface, even though an SLA configuration without an outcome isn't persisted and doesn't make business sense.\n\nThe Action Center backend team had added a validation to throw a bad request (400) in case of a non null SLA object with a missing outcome. This ended up impacting customers who had added the Start/End criteria on the node but not configured an outcome. Instances that were previously running failed with 400s, and were unable to resume\n\n## Detection\n\nA customer reported the issue to Maestro Engineering SME, who raised it internally with the team. The Maestro team investigated this failure, and observed an increasing number of failures for USI, and a few other tenants via logs, leading them to engage the Action Center team.\n\n## Response\n\nThe Maestro team received report of the issue from the customer via Email, and they did some initial investigation and identified a potential regression, following which, the Action Center SME was engaged, the issue reproduced on lower environments, a rollback of the feature applied and tested, and rolled out to all impacted regions. It was resolved with all instances being internally retried and resuming.\n\n## Follow up\n\n1. Add validation for this scenario across all cross service codebase surfaces to ensure human review\n2. Add required monitoring and integration tests in place to detect this issue faster",
"postmortem_body_last_updated_at": "2026-10-26T03:16:04.021Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-26T03:16:04.021Z",
"reminder_intervals": null,
"resolved_at": "2026-10-08T04:29:22.398Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/4h2m6gdys1yq",
"started_at": "2026-10-07T21:34:37.189Z",
"status": "resolved",
"updated_at": "2026-10-26T03:16:04.161Z"
},
{
"components": [
{
"created_at": "2026-03-01T05:02:17.880Z",
"description": "Orchestrator in United States",
"group_id": "8md89tvvp0jc",
"id": "8c2g6j81glkx",
"name": "Orchestrator",
"page_id": "rtthtlr99830",
"position": 2,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-07T19:06:26.394Z"
}
],
"created_at": "2026-10-07T17:37:47.336Z",
"id": "csffj80jkmlr",
"impact": "minor",
"impact_override": "minor",
"incident_updates": [
{
"affected_components": [
{
"code": "8c2g6j81glkx",
"name": "United States - Orchestrator",
"new_status": "operational",
"old_status": "degraded_performance"
}
],
"body": "We have the issue in the East US region and error rates and latency returned to normal levels.",
"created_at": "2026-10-07T19:06:26.441Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-07T19:06:26.441Z",
"id": "qkw25p3r8ccc",
"incident_id": "csffj80jkmlr",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-07T19:06:26.441Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "8c2g6j81glkx",
"name": "United States - Orchestrator",
"new_status": "degraded_performance",
"old_status": "degraded_performance"
}
],
"body": "We have applied a capacity increase to the affected database infrastructure in the East US region and error rates and latency returned to normal levels. We will provide an update as soon as more information is available.",
"created_at": "2026-10-07T18:13:28.110Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-07T18:13:28.110Z",
"id": "ntrpw1nj5kx4",
"incident_id": "csffj80jkmlr",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-07T18:13:28.110Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "8c2g6j81glkx",
"name": "United States - Orchestrator",
"new_status": "degraded_performance",
"old_status": "operational"
}
],
"body": "We are investigating reports of intermittent slowness and errors affecting a subset of organizations in the East US region of Automation Cloud. Affected users may see pages in the Orchestrator UI fail to load (for example, the Jobs page returning \"An error has occurred\"), and robot operations such as queue transactions and session starts may intermittently time out. We will provide an update as soon as more information is available.",
"created_at": "2026-10-07T17:37:47.441Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-07T17:37:47.441Z",
"id": "88j3gr0dftgt",
"incident_id": "csffj80jkmlr",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-07T17:37:47.441Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-07T18:13:28.110Z",
"name": "US - Orchestrator - Intermittent errors and degraded performance in Orchestrator",
"page_id": "rtthtlr99830",
"postmortem_body": "## Customer impact\n\nBetween approximately 13:00 UTC and 18:15 UTC on October 7, 2026, customers in the US region experienced intermittent errors and slow responses in Orchestrator. Affected users may have seen Orchestrator pages fail to load, for example the Jobs page returning “An error has occurred,” and robot operations such as queue transactions and session starts may have intermittently timed out. Other UiPath services that depend on Orchestrator may also have seen slower responses or errors during this period.\n\n## Root cause\n\nSeveral frequently-used Orchestrator operations each relied on a shared short-lived resource inside the database, including the routine status check that every running robot performs. As the volume of concurrent robot activity in this region grew, requests began queueing for access to that shared resource instead of completing their work. \nThe effect was selective rather than a general slowdown: the operations that used this resource were delayed, while others continued to run normally. Because the database’s processor, memory and storage usage all remained within normal ranges, routine capacity monitoring did not indicate a problem.\n\n## Detection\n\nOur automated monitoring detected the degradation within approximately 35 minutes of customer impact beginning, and our engineers identified the underlying contention during the event. Because this was a contention problem rather than a shortage of capacity, and all capacity indicators were normal, increasing database capacity was not expected to resolve it, and the team investigated alternatives before applying it. That assessment delayed the mitigation.\n\n## Response\n\nOur engineering team increased the capacity of the affected database. Response times and error rates returned to normal immediately, and we continued monitoring to confirm the service was stable before closing the incident.\n\n* 13:00 : Error rates and response times begin to rise for customers in the US region. \n* 13:36 : Automated monitoring detects the degradation and notifies the on-call engineering team. \n* 13:36 – 17:29 : Engineers investigate. All standard database capacity indicators — processor, memory and storage — remain within normal ranges throughout, so the cause is not apparent from routine monitoring. \n* 17:29 : A formal incident is declared and the full response team is engaged. 17:38 A public notice is published on [UiPath Status](http://status.uipath.com/) \\(Investigating\\). \n* 17:31 – 17:43 : Additional database capacity is provisioned for the affected region. \n* 18:00 : Error rates, response times and request latency return to normal levels. \n* 18:11 : The status page is updated to Monitoring \u0026 Customer impact ends. \n* 19:06 : The incident is closed after continuous monitoring confirms the service is stable.\n\n## Follow up\n\nTo prevent this from happening again, we are:\n\n* Changing the affected operations so they no longer depend on the shared resource. This has been rolled out to all affected systems and has substantially reduced the condition that caused the incident.\n* Enabling detailed database monitoring on these systems, so this class of contention is visible while it is happening rather than afterwards.\n* Improving our alerting so that a genuine degradation of this kind is clearly distinguishable from routine, self-correcting variation, and is escalated faster.",
"postmortem_body_last_updated_at": "2026-10-14T11:46:51.000Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-14T11:42:25.653Z",
"reminder_intervals": null,
"resolved_at": "2026-10-07T19:06:26.441Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/bp4fkvjf990h",
"started_at": "2026-10-07T17:37:47.328Z",
"status": "resolved",
"updated_at": "2026-10-14T11:46:51.003Z"
},
{
"components": [
{
"created_at": "2021-10-18T17:41:13.208Z",
"description": null,
"group_id": "bt4fyv3z54lv",
"id": "7fp5ydxz32r2",
"name": "Orchestrator",
"page_id": "rtthtlr99830",
"position": 2,
"showcase": true,
"start_date": "2021-01-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-06T18:44:43.054Z"
},
{
"created_at": "2026-03-01T05:03:11.151Z",
"description": "Orchestrator in Japan",
"group_id": "q9rqbbwgxqfz",
"id": "cpvmj871ptxl",
"name": "Orchestrator",
"page_id": "rtthtlr99830",
"position": 2,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-13T10:57:35.803Z"
},
{
"created_at": "2026-03-01T05:05:40.769Z",
"description": "Orchestrator in Australia",
"group_id": "0h50tbzgz6n8",
"id": "07mgmfr738cj",
"name": "Orchestrator",
"page_id": "rtthtlr99830",
"position": 2,
"showcase": true,
"start_date": "2026-03-01T00:00:00.000Z",
"status": "operational",
"updated_at": "2026-10-06T18:44:43.119Z"
}
],
"created_at": "2026-10-06T17:50:52.622Z",
"id": "cd9kf5vgplmg",
"impact": "major",
"impact_override": "major",
"incident_updates": [
{
"affected_components": [
{
"code": "7fp5ydxz32r2",
"name": "European Union - Orchestrator",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "cpvmj871ptxl",
"name": "Japan - Orchestrator",
"new_status": "operational",
"old_status": "partial_outage"
},
{
"code": "07mgmfr738cj",
"name": "Australia - Orchestrator",
"new_status": "operational",
"old_status": "partial_outage"
}
],
"body": "We have mitigated the issue impacting ingestion of robot logs for Orchestrator in Europe.\nImpact: Users may expect delays for ingestion of robot logs and/or search/display of the logs. No logs will be lost.\nNext update: Our teams are working to understand the cause and scope and will share updates as available.",
"created_at": "2026-10-06T18:44:43.322Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T18:44:43.322Z",
"id": "561d0cq1jxd1",
"incident_id": "cd9kf5vgplmg",
"status": "resolved",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T18:44:43.322Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "7fp5ydxz32r2",
"name": "European Union - Orchestrator",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "cpvmj871ptxl",
"name": "Japan - Orchestrator",
"new_status": "partial_outage",
"old_status": "operational"
},
{
"code": "07mgmfr738cj",
"name": "Australia - Orchestrator",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We have mitigated the issue impacting ingestion of robot logs for Orchestrator in Europe.\nImpact: Users may expect delays for ingestion of robot logs and/or search/display of the logs. No logs will be lost.\nNext update: Our teams are working to understand the cause and scope and will share updates as available.",
"created_at": "2026-10-06T18:43:27.798Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T18:43:27.798Z",
"id": "xzg83yl2rkcy",
"incident_id": "cd9kf5vgplmg",
"status": "monitoring",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T18:43:27.798Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "7fp5ydxz32r2",
"name": "European Union - Orchestrator",
"new_status": "partial_outage",
"old_status": "partial_outage"
},
{
"code": "cpvmj871ptxl",
"name": "Japan - Orchestrator",
"new_status": "operational",
"old_status": "operational"
},
{
"code": "07mgmfr738cj",
"name": "Australia - Orchestrator",
"new_status": "operational",
"old_status": "operational"
}
],
"body": "We have identified the cause for an outage impacting ingestion of robot logs for Orchestrator in Europe.\nImpact: Users may expect delays for ingestion of robot logs and/or search/display of the logs. No logs will be lost.\nNext update: Our teams are working to understand the cause and scope and will share updates as available.",
"created_at": "2026-10-06T18:28:12.125Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T18:28:12.125Z",
"id": "g6bbmplxl5wd",
"incident_id": "cd9kf5vgplmg",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T18:28:12.125Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "7fp5ydxz32r2",
"name": "European Union - Orchestrator",
"new_status": "partial_outage",
"old_status": "partial_outage"
}
],
"body": "We have identified the cause for an outage impacting ingestion of robot logs for Orchestrator in Europe.\nImpact: Users may expect delays for ingestion of robot logs and/or search/display of the logs. No logs will be lost.\nNext update: Our teams are working to understand the cause and scope and will share updates as available.",
"created_at": "2026-10-06T18:02:57.482Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T18:02:57.482Z",
"id": "dcvyg2pqx2cz",
"incident_id": "cd9kf5vgplmg",
"status": "identified",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T18:02:57.482Z",
"wants_twitter_update": false
},
{
"affected_components": [
{
"code": "7fp5ydxz32r2",
"name": "European Union - Orchestrator",
"new_status": "partial_outage",
"old_status": "operational"
}
],
"body": "We are investigating reports of an outage impacting [impacted functionality or user symptom] for Orchestrator in Europe.\nImpact: Users may expect delays for ingestion of robot logs and/or search/display of the logs. No logs will be lost.\nNext update: Our teams are working to understand the cause and scope and will share updates as available.",
"created_at": "2026-10-06T17:50:52.729Z",
"custom_tweet": null,
"deliver_notifications": true,
"display_at": "2026-10-06T17:50:52.729Z",
"id": "mx6n13qcnhnh",
"incident_id": "cd9kf5vgplmg",
"status": "investigating",
"tweet_id": null,
"twitter_updated_at": null,
"updated_at": "2026-10-06T17:50:52.729Z",
"wants_twitter_update": false
}
],
"metadata": {},
"monitoring_at": "2026-10-06T18:43:27.798Z",
"name": "[Europe, Japan, Australia] - [Orchestrator] - [Delays for ingestion of robot logs and search or display of the logs]",
"page_id": "rtthtlr99830",
"postmortem_body": "## Customer impact\n\nOn , between approximately 15:45 UTC and 18:45 UTC, customers hosted in the Europe, Australia and Japan regions were unable to see recent robot logs in Orchestrator. Logs written during the impact window did not appear in the Jobs and Logs views or through the Robot Logs API, and logs older than approximately 03:00 UTC that day were available only intermittently. The impact window differed per region:\n\n- Australia: approximately 15:45 UTC to 18:35 UTC\n- Europe: approximately 16:35 UTC to 18:00 UTC\n- Japan: approximately 16:45 UTC to 18:30 UTC\n\nJob execution, queues, triggers and all other Orchestrator functionality were not affected. No robot logs were lost. Logs produced during the impact window continued to be collected and became visible again once the issue was mitigated.\n\n## Root cause\n\nWe were performing a scheduled upgrade of the Elasticsearch service that stores Orchestrator robot logs. The upgrade uses a blue/green approach: a new cluster is created alongside the existing one, the data is restored into it from a backup, the pipeline that ingests new logs is started on it, and only then is traffic switched over.\n\nIn the affected regions, the traffic switch was approved by the engineer running the upgrade before the data restore and the ingestion catch-up had completed, which is a deviation from our upgrade procedure. This was a human error. As a result, Orchestrator was reading robot logs from a cluster that did not yet contain recent data, while new logs continued to be collected but were not yet indexed on the cluster serving requests. The error was not caught because our deployment tooling relies on the operator to verify data completeness and does not enforce the order between the data restore and the traffic switch.\n\n## Detection\n\nOur synthetic monitoring, which continuously runs test jobs and verifies that their logs are returned, detected missing logs in the affected regions and raised alerts. Our engineering team correlated the alerts with the in-progress upgrade, declared an incident and published a status page notification.\n\n## Response\n\nOur engineering team reverted the traffic to the previous Elasticsearch clusters in all three regions, which immediately restored access to all robot logs, including those produced during the impact window. We confirmed recovery through synthetic monitoring in every affected region and continued monitoring before marking the incident resolved at approximately 18:45 UTC.\n\nThe upgrade of the new clusters continues in the background. Traffic will be moved to them only after the data restore is complete and ingestion is confirmed to be up to date.\n\n## Follow up\n\nTo prevent this from happening again, we are:\n\n- Changing the deployment pipeline so that the traffic switch cannot run until the data restore and log ingestion on the new cluster are verified as complete.\n- Adding an automated rollback step to the pipeline, so traffic can be returned to the previous cluster within minutes if a problem is detected after a switch.",
"postmortem_body_last_updated_at": "2026-10-07T11:08:34.322Z",
"postmortem_ignored": false,
"postmortem_notified_subscribers": true,
"postmortem_notified_twitter": false,
"postmortem_published_at": "2026-10-07T11:08:34.322Z",
"reminder_intervals": null,
"resolved_at": "2026-10-06T18:44:43.322Z",
"scheduled_auto_completed": false,
"scheduled_auto_in_progress": false,
"scheduled_for": null,
"scheduled_remind_prior": false,
"scheduled_reminded_at": null,
"scheduled_until": null,
"shortlink": "https://stspg.io/k4f6hv1cbjj8",
"started_at": "2026-10-06T17:50:52.609Z",
"status": "resolved",
"updated_at": "2026-10-07T11:08:34.425Z"
}
],
"page": {
"id": "rtthtlr99830",
"name": "UiPath",
"url": "https://status.uipath.com"
},
"status": {
"description": "All Systems Operational",
"indicator": "none"
}
}