|
2022-05-24
ยง
|
| 12:17 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti2025.codfw.wmnet |
[production] |
| 12:16 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1161 (T298560)', diff saved to https://phabricator.wikimedia.org/P28448 and previous config saved to /var/cache/conftool/dbconfig/20220524-121641-ladsgroup.json |
[production] |
| 12:16 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on clouddb[1016,1020-1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:16 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on clouddb[1016,1020-1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:16 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1161.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:16 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1161.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:16 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1144:3315 (T298560)', diff saved to https://phabricator.wikimedia.org/P28447 and previous config saved to /var/cache/conftool/dbconfig/20220524-121627-ladsgroup.json |
[production] |
| 12:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1118 (T303603)', diff saved to https://phabricator.wikimedia.org/P28446 and previous config saved to /var/cache/conftool/dbconfig/20220524-120816-ladsgroup.json |
[production] |
| 12:06 |
<jbond> |
disable puppet on c:httpd |
[production] |
| 12:04 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti2024.codfw.wmnet |
[production] |
| 12:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1144:3315', diff saved to https://phabricator.wikimedia.org/P28445 and previous config saved to /var/cache/conftool/dbconfig/20220524-120122-ladsgroup.json |
[production] |
| 11:52 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1118 (T303603)', diff saved to https://phabricator.wikimedia.org/P28444 and previous config saved to /var/cache/conftool/dbconfig/20220524-115251-ladsgroup.json |
[production] |
| 11:52 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1118.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:52 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1118.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:52 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134 (T303603)', diff saved to https://phabricator.wikimedia.org/P28443 and previous config saved to /var/cache/conftool/dbconfig/20220524-115243-ladsgroup.json |
[production] |
| 11:46 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1144:3315', diff saved to https://phabricator.wikimedia.org/P28442 and previous config saved to /var/cache/conftool/dbconfig/20220524-114617-ladsgroup.json |
[production] |
| 11:45 |
<jbond> |
disable puppet on mw servers |
[production] |
| 11:40 |
<jbond@cumin1001> |
END (ERROR) - Cookbook sre.hosts.reboot-single (exit_code=97) for host puppetmaster1001.eqiad.wmnet |
[production] |
| 11:37 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134', diff saved to https://phabricator.wikimedia.org/P28441 and previous config saved to /var/cache/conftool/dbconfig/20220524-113738-ladsgroup.json |
[production] |
| 11:34 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster1001.eqiad.wmnet |
[production] |
| 11:33 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti2024.codfw.wmnet |
[production] |
| 11:31 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1144:3315 (T298560)', diff saved to https://phabricator.wikimedia.org/P28440 and previous config saved to /var/cache/conftool/dbconfig/20220524-113112-ladsgroup.json |
[production] |
| 11:30 |
<jbond> |
disable puppet fleet wide |
[production] |
| 11:23 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 11:22 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134', diff saved to https://phabricator.wikimedia.org/P28439 and previous config saved to /var/cache/conftool/dbconfig/20220524-112233-ladsgroup.json |
[production] |
| 11:19 |
<elukey@cumin1001> |
START - Cookbook sre.dns.netbox |
[production] |
| 11:19 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti2023.codfw.wmnet |
[production] |
| 11:14 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti2023.codfw.wmnet |
[production] |
| 11:07 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134 (T303603)', diff saved to https://phabricator.wikimedia.org/P28438 and previous config saved to /var/cache/conftool/dbconfig/20220524-110728-ladsgroup.json |
[production] |
| 11:00 |
<jynus> |
restart db1150 T308315 |
[production] |
| 10:51 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1134 (T303603)', diff saved to https://phabricator.wikimedia.org/P28437 and previous config saved to /var/cache/conftool/dbconfig/20220524-105116-ladsgroup.json |
[production] |
| 10:51 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1134.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:51 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1134.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:51 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1135 (T303603)', diff saved to https://phabricator.wikimedia.org/P28436 and previous config saved to /var/cache/conftool/dbconfig/20220524-105108-ladsgroup.json |
[production] |
| 10:45 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti2022.codfw.wmnet |
[production] |
| 10:43 |
<jbond@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster2005.codfw.wmnet |
[production] |
| 10:43 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster2004.codfw.wmnet |
[production] |
| 10:39 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster2005.codfw.wmnet |
[production] |
| 10:38 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster2004.codfw.wmnet |
[production] |
| 10:38 |
<jbond@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster2003.codfw.wmnet |
[production] |
| 10:38 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster1005.eqiad.wmnet |
[production] |
| 10:37 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster1003.eqiad.wmnet |
[production] |
| 10:36 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1135', diff saved to https://phabricator.wikimedia.org/P28435 and previous config saved to /var/cache/conftool/dbconfig/20220524-103603-ladsgroup.json |
[production] |
| 10:33 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster1003.eqiad.wmnet |
[production] |
| 10:33 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster2003.codfw.wmnet |
[production] |
| 10:33 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster1002.eqiad.wmnet |
[production] |
| 10:32 |
<jbond@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster2002.codfw.wmnet |
[production] |
| 10:32 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster1005.eqiad.wmnet |
[production] |
| 10:32 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster1004.eqiad.wmnet |
[production] |
| 10:28 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster1004.eqiad.wmnet |
[production] |