|
2024-11-04
ยง
|
| 14:10 |
<aborrero@cloudcumin1001> |
START - Cookbook wmcs.openstack.restart_openstack |
[admin] |
| 14:10 |
<ladsgroup@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1172', diff saved to https://phabricator.wikimedia.org/P70868 and previous config saved to /var/cache/conftool/dbconfig/20241104-141023-ladsgroup.json |
[production] |
| 14:09 |
<aborrero@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.openstack.restart_openstack (exit_code=99) |
[admin] |
| 14:09 |
<aborrero@cloudcumin1001> |
START - Cookbook wmcs.openstack.restart_openstack |
[admin] |
| 14:08 |
<aborrero@cloudcumin2001> |
END (FAIL) - Cookbook wmcs.openstack.restart_openstack (exit_code=99) |
[admin] |
| 14:08 |
<aborrero@cloudcumin2001> |
START - Cookbook wmcs.openstack.restart_openstack |
[admin] |
| 14:08 |
<raymond-ndibe@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=99) for node tools-k8s-worker-nfs-58 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 14:08 |
<aborrero@cloudcumin2001> |
END (PASS) - Cookbook wmcs.openstack.restart_openstack (exit_code=0) |
[admin] |
| 14:08 |
<aborrero@cloudcumin2001> |
START - Cookbook wmcs.openstack.restart_openstack |
[admin] |
| 14:02 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-58 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 14:02 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-57 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 14:01 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-57 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 14:01 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-55 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 14:00 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-55 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 14:00 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-54 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:59 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-54 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:59 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-53 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:57 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-53 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:57 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-50 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:56 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-50 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:56 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-5 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:55 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-5 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:55 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-48 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:55 |
<ladsgroup@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1172 (T376905)', diff saved to https://phabricator.wikimedia.org/P70867 and previous config saved to /var/cache/conftool/dbconfig/20241104-135516-ladsgroup.json |
[production] |
| 13:54 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-48 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:54 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-47 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:53 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-47 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:53 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-46 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:52 |
<arturo> |
[codfw1dev] live-hack cloudlb2001-dev and cloudcontrol2004-dev for T378192 |
[admin] |
| 13:52 |
<arturo> |
[codfw1dev] restart rabbitmq |
[admin] |
| 13:51 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-46 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:51 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-45 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:51 |
<marostegui> |
Start schema change on redacteddb1001:s8 T367856 (this will make replication in s8 lag for around 2-3 days) |
[production] |
| 13:50 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-45 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:50 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-44 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:50 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3 days, 0:00:00 on an-redacteddb1001.eqiad.wmnet with reason: Schema change T367856 |
[production] |
| 13:50 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 3 days, 0:00:00 on an-redacteddb1001.eqiad.wmnet with reason: Schema change T367856 |
[production] |
| 13:49 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-44 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:49 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-43 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:48 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-43 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:48 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-42 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:47 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-42 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:47 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-41 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:46 |
<ladsgroup@cumin1002> |
dbctl commit (dc=all): 'Depooling db1172 (T376905)', diff saved to https://phabricator.wikimedia.org/P70866 and previous config saved to /var/cache/conftool/dbconfig/20241104-134605-ladsgroup.json |
[production] |
| 13:46 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-41 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:46 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-40 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:45 |
<ladsgroup@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1172.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:45 |
<ladsgroup@cumin1002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1172.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:44 |
<raymond-ndibe@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-nfs-40 from 1.27.16 to 1.28.14 (T362867) |
[tools] |
| 13:44 |
<raymond-ndibe@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-nfs-39 from 1.27.16 to 1.28.14 (T362867) |
[tools] |