|
2021-02-10
ยง
|
| 10:39 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp3052.esams.wmnet |
[production] |
| 10:39 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp2030.codfw.wmnet |
[production] |
| 10:38 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp3053.esams.wmnet |
[production] |
| 10:38 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp2029.codfw.wmnet |
[production] |
| 10:37 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp1078.eqiad.wmnet |
[production] |
| 10:37 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp1077.eqiad.wmnet |
[production] |
| 10:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 80%: Slowly repool db1127', diff saved to https://phabricator.wikimedia.org/P14300 and previous config saved to /var/cache/conftool/dbconfig/20210210-103146-root.json |
[production] |
| 10:28 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp5002.eqsin.wmnet |
[production] |
| 10:28 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp5008.eqsin.wmnet |
[production] |
| 10:28 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp4022.ulsfo.wmnet |
[production] |
| 10:27 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp4028.ulsfo.wmnet |
[production] |
| 10:27 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp3053.esams.wmnet |
[production] |
| 10:26 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp3052.esams.wmnet |
[production] |
| 10:26 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp2030.codfw.wmnet |
[production] |
| 10:25 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp2029.codfw.wmnet |
[production] |
| 10:25 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp1078.eqiad.wmnet |
[production] |
| 10:25 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp1077.eqiad.wmnet |
[production] |
| 10:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 60%: Slowly repool db1127', diff saved to https://phabricator.wikimedia.org/P14299 and previous config saved to /var/cache/conftool/dbconfig/20210210-101642-root.json |
[production] |
| 10:16 |
<moritzm> |
installing firejail security updates |
[production] |
| 10:14 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) |
[production] |
| 10:10 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp4021.ulsfo.wmnet |
[production] |
| 10:05 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.decommission |
[production] |
| 10:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 40%: Slowly repool db1127', diff saved to https://phabricator.wikimedia.org/P14298 and previous config saved to /var/cache/conftool/dbconfig/20210210-100139-root.json |
[production] |
| 10:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1076 (re)pooling @ 100%: Slowly repooling db1076 after cloning db1162', diff saved to https://phabricator.wikimedia.org/P14297 and previous config saved to /var/cache/conftool/dbconfig/20210210-100111-root.json |
[production] |
| 10:00 |
<vgutierrez> |
power cycling cp4021 |
[production] |
| 09:57 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp5007.eqsin.wmnet |
[production] |
| 09:56 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp5001.eqsin.wmnet |
[production] |
| 09:50 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp4027.ulsfo.wmnet |
[production] |
| 09:48 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp3051.esams.wmnet |
[production] |
| 09:47 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp3050.esams.wmnet |
[production] |
| 09:47 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp2027.codfw.wmnet |
[production] |
| 09:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 20%: Slowly repool db1127', diff saved to https://phabricator.wikimedia.org/P14296 and previous config saved to /var/cache/conftool/dbconfig/20210210-094635-root.json |
[production] |
| 09:46 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp2028.codfw.wmnet |
[production] |
| 09:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1076 (re)pooling @ 75%: Slowly repooling db1076 after cloning db1162', diff saved to https://phabricator.wikimedia.org/P14295 and previous config saved to /var/cache/conftool/dbconfig/20210210-094608-root.json |
[production] |
| 09:46 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp1075.eqiad.wmnet |
[production] |
| 09:45 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp1076.eqiad.wmnet |
[production] |
| 09:40 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp5001.eqsin.wmnet |
[production] |
| 09:40 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp5007.eqsin.wmnet |
[production] |
| 09:38 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp4021.ulsfo.wmnet |
[production] |
| 09:38 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp4027.ulsfo.wmnet |
[production] |
| 09:37 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp3051.esams.wmnet |
[production] |
| 09:36 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp3050.esams.wmnet |
[production] |
| 09:35 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp2028.codfw.wmnet |
[production] |
| 09:34 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp2027.codfw.wmnet |
[production] |
| 09:33 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp1076.eqiad.wmnet |
[production] |
| 09:31 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp1075.eqiad.wmnet |
[production] |
| 09:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 10%: Slowly repool db1127', diff saved to https://phabricator.wikimedia.org/P14294 and previous config saved to /var/cache/conftool/dbconfig/20210210-093132-root.json |
[production] |
| 09:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1076 (re)pooling @ 50%: Slowly repooling db1076 after cloning db1162', diff saved to https://phabricator.wikimedia.org/P14293 and previous config saved to /var/cache/conftool/dbconfig/20210210-093104-root.json |
[production] |
| 09:30 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 100%: Slowly repool db1127', diff saved to https://phabricator.wikimedia.org/P14292 and previous config saved to /var/cache/conftool/dbconfig/20210210-093011-root.json |
[production] |
| 09:23 |
<vgutierrez> |
rolling restart of cp nodes to catch up on kernel upgrades |
[production] |