2023-10-31
§
|
23:46 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
18:41 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
14:12 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) (348643) |
[admin] |
14:11 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (348643) |
[admin] |
14:10 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
10:16 |
<dhinus> |
upgrading mariadb-server in cloudcontrol1005 (T345811) |
[admin] |
10:04 |
<dhinus> |
upgrading mariadb-server in cloudcontrol1006 (T345811) |
[admin] |
09:51 |
<dhinus> |
upgrading mariadb-server in cloudcontrol1007, second attempt (T345811) |
[admin] |
06:16 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.drain_node (exit_code=99) (348643) |
[admin] |
05:20 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
02:27 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) (348643) |
[admin] |
02:27 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (348643) |
[admin] |
02:27 |
<andrew@cloudcumin1001> |
END (ERROR) - Cookbook wmcs.ceph.osd.drain_node (exit_code=97) (348643) |
[admin] |
02:26 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
01:59 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
01:59 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
01:07 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
01:07 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
01:07 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
2023-10-30
§
|
17:40 |
<andrewbogott> |
rebooting tools-db-1.tools.eqiad1.wikimedia.cloud for yet another oom death |
[admin] |
17:09 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
17:04 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
17:02 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
16:56 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.drain_node (exit_code=99) (348643) |
[admin] |
16:55 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.drain_node (exit_code=99) (348643) |
[admin] |
16:55 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
16:55 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
16:54 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) |
[admin] |
16:53 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |
16:53 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) |
[admin] |
16:52 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |
16:52 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) |
[admin] |
16:51 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |
16:39 |
<dhinus> |
upgrading mariadb-server in cloudcontrol1007 (T345811) |
[admin] |
16:38 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
16:38 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
16:37 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
16:37 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
16:37 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
16:37 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
16:30 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
16:30 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
16:30 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
16:30 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
16:29 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
16:29 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
16:17 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) |
[admin] |
16:17 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |
16:17 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) |
[admin] |
16:17 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |