|
2023-10-31
§
|
| 23:46 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 18:41 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
| 14:12 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) (348643) |
[admin] |
| 14:11 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (348643) |
[admin] |
| 14:10 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 10:16 |
<dhinus> |
upgrading mariadb-server in cloudcontrol1005 (T345811) |
[admin] |
| 10:04 |
<dhinus> |
upgrading mariadb-server in cloudcontrol1006 (T345811) |
[admin] |
| 09:51 |
<dhinus> |
upgrading mariadb-server in cloudcontrol1007, second attempt (T345811) |
[admin] |
| 06:16 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.drain_node (exit_code=99) (348643) |
[admin] |
| 05:20 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
| 02:27 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) (348643) |
[admin] |
| 02:27 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (348643) |
[admin] |
| 02:27 |
<andrew@cloudcumin1001> |
END (ERROR) - Cookbook wmcs.ceph.osd.drain_node (exit_code=97) (348643) |
[admin] |
| 02:26 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 01:59 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 01:59 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 01:07 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
| 01:07 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
| 01:07 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.drain_node (exit_code=0) (348643) |
[admin] |
|
2023-10-30
§
|
| 17:40 |
<andrewbogott> |
rebooting tools-db-1.tools.eqiad1.wikimedia.cloud for yet another oom death |
[admin] |
| 17:09 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 17:04 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 17:02 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 16:56 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.drain_node (exit_code=99) (348643) |
[admin] |
| 16:55 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.drain_node (exit_code=99) (348643) |
[admin] |
| 16:55 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 16:55 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.drain_node (348643) |
[admin] |
| 16:54 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) |
[admin] |
| 16:53 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |
| 16:53 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) |
[admin] |
| 16:52 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |
| 16:52 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=0) |
[admin] |
| 16:51 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |
| 16:39 |
<dhinus> |
upgrading mariadb-server in cloudcontrol1007 (T345811) |
[admin] |
| 16:38 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
| 16:38 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
| 16:37 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
| 16:37 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
| 16:37 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
| 16:37 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
| 16:30 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
| 16:30 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
| 16:30 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
| 16:30 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
| 16:29 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) (T348643) |
[admin] |
| 16:29 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node (T348643) |
[admin] |
| 16:17 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) |
[admin] |
| 16:17 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |
| 16:17 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.ceph.osd.undrain_node (exit_code=99) |
[admin] |
| 16:17 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.ceph.osd.undrain_node |
[admin] |