|
2022-04-20
ยง
|
| 17:46 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1021.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:46 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1019.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:46 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1013.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:46 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1022.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:46 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1016.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:45 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1015.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:45 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'es1028 (re)pooling @ 50%: Reboot T303174', diff saved to https://phabricator.wikimedia.org/P25790 and previous config saved to /var/cache/conftool/dbconfig/20220420-174508-kormat.json |
[production] |
| 17:40 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1023.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:40 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1012.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:39 |
<cmjohnson@cumin1001> |
END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host parse1020.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:39 |
<cmjohnson@cumin1001> |
END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host parse1015.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:39 |
<cmjohnson@cumin1001> |
END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host parse1019.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175', diff saved to https://phabricator.wikimedia.org/P25789 and previous config saved to /var/cache/conftool/dbconfig/20220420-173814-ladsgroup.json |
[production] |
| 17:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1161 (T298565)', diff saved to https://phabricator.wikimedia.org/P25788 and previous config saved to /var/cache/conftool/dbconfig/20220420-173405-ladsgroup.json |
[production] |
| 17:34 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on clouddb[1016,1020-1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:34 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on clouddb[1016,1020-1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:34 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1161.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:34 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1161.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:34 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1019.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:34 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1018.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:34 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1016.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:34 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1021.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:34 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1022.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:34 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1020.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:33 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1014.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:33 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1015.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:33 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1017.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:33 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1013.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:33 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161 (T298565)', diff saved to https://phabricator.wikimedia.org/P25787 and previous config saved to /var/cache/conftool/dbconfig/20220420-173304-ladsgroup.json |
[production] |
| 17:32 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'es1025 (re)pooling @ 25%: Reboot T303174', diff saved to https://phabricator.wikimedia.org/P25786 and previous config saved to /var/cache/conftool/dbconfig/20220420-173207-kormat.json |
[production] |
| 17:31 |
<cmjohnson@cumin1001> |
END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host parse1003.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:31 |
<dzahn@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on gitlab-runner2001.codfw.wmnet with reason: reimage |
[production] |
| 17:31 |
<dzahn@cumin2002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on gitlab-runner2001.codfw.wmnet with reason: reimage |
[production] |
| 17:31 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1007.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:31 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1004.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:31 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1010.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:31 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1006.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:31 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1002.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:30 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1009.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:30 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1011.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:30 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1005.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:30 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1008.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:30 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'es1028 (re)pooling @ 25%: Reboot T303174', diff saved to https://phabricator.wikimedia.org/P25785 and previous config saved to /var/cache/conftool/dbconfig/20220420-173004-kormat.json |
[production] |
| 17:27 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.provision for host parse1012.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:26 |
<kormat@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:30:00 on es1025.eqiad.wmnet with reason: Rebooting for T303174 |
[production] |
| 17:26 |
<kormat@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:30:00 on es1025.eqiad.wmnet with reason: Rebooting for T303174 |
[production] |
| 17:26 |
<kormat@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:30:00 on es1028.eqiad.wmnet with reason: Rebooting for T303174 |
[production] |
| 17:26 |
<kormat@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:30:00 on es1028.eqiad.wmnet with reason: Rebooting for T303174 |
[production] |
| 17:26 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host parse1001.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 17:23 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175', diff saved to https://phabricator.wikimedia.org/P25784 and previous config saved to /var/cache/conftool/dbconfig/20220420-172309-ladsgroup.json |
[production] |