|
2021-02-01
ยง
|
| 17:03 |
<jbond@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host idp1001.wikimedia.org |
[production] |
| 17:01 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host idp1001.wikimedia.org |
[production] |
| 17:01 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host apt1001.wikimedia.org |
[production] |
| 16:53 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host idp-test2001.wikimedia.org |
[production] |
| 16:52 |
<jbond@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host idp2001.wikimedia.org |
[production] |
| 16:50 |
<jbond@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host apt2001.wikimedia.org |
[production] |
| 16:46 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host idp2001.wikimedia.org |
[production] |
| 16:46 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host apt2001.wikimedia.org |
[production] |
| 16:44 |
<jbond@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host pki1001.eqiad.wmnet |
[production] |
| 16:44 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host pki2001.codfw.wmnet |
[production] |
| 16:43 |
<jbond@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host idp-test1001.wikimedia.org |
[production] |
| 16:42 |
<jbond42> |
enable puppet fleet wide to post reboots |
[production] |
| 16:35 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host idp-test2001.wikimedia.org |
[production] |
| 16:35 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host idp-test1001.wikimedia.org |
[production] |
| 16:34 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host pki1001.eqiad.wmnet |
[production] |
| 16:34 |
<jbond@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster2003.codfw.wmnet |
[production] |
| 16:34 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host pki2001.codfw.wmnet |
[production] |
| 16:33 |
<jbond@cumin1001> |
END (ERROR) - Cookbook sre.hosts.reboot-single (exit_code=97) for host puppetmaster2001.codfw.wmnet |
[production] |
| 16:33 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster2002.codfw.wmnet |
[production] |
| 16:33 |
<jbond@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetdb2002.codfw.wmnet |
[production] |
| 16:28 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host puppetdb2002.codfw.wmnet |
[production] |
| 16:28 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster2003.codfw.wmnet |
[production] |
| 16:28 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster2002.codfw.wmnet |
[production] |
| 16:28 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster2001.codfw.wmnet |
[production] |
| 16:28 |
<jbond@cumin1001> |
END (ERROR) - Cookbook sre.hosts.reboot-single (exit_code=97) for host puppetmaster1001.eqiad.wmnet |
[production] |
| 16:26 |
<jbond@cumin2001> |
END (ERROR) - Cookbook sre.hosts.reboot-single (exit_code=97) for host puppetdb1002.eqiad.wmnet |
[production] |
| 16:22 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster1002.eqiad.wmnet |
[production] |
| 16:21 |
<jbond@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetmaster1003.eqiad.wmnet |
[production] |
| 16:15 |
<XioNoX> |
fail-back RG1 back to node1 on pfw3-eqiad - T263833 |
[production] |
| 16:14 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster1001.eqiad.wmnet |
[production] |
| 16:14 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster1002.eqiad.wmnet |
[production] |
| 16:14 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host puppetmaster1003.eqiad.wmnet |
[production] |
| 16:13 |
<jbond@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host puppetdb1002.eqiad.wmnet |
[production] |
| 16:12 |
<jbond42> |
disable puppet fleet wide to preform reboots |
[production] |
| 16:09 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetboard1002.eqiad.wmnet |
[production] |
| 16:05 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetboard1002.eqiad.wmnet |
[production] |
| 16:05 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetboard2002.codfw.wmnet |
[production] |
| 16:03 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetboard2002.codfw.wmnet |
[production] |
| 16:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1147 (re)pooling @ 100%: Repool db1147 after a restart', diff saved to https://phabricator.wikimedia.org/P14110 and previous config saved to /var/cache/conftool/dbconfig/20210201-160122-root.json |
[production] |
| 15:59 |
<jbond42> |
install buster kernel update |
[production] |
| 15:46 |
<XioNoX> |
failover RG1 back to node0 on pfw3-eqiad - T263833 |
[production] |
| 15:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1147 (re)pooling @ 80%: Repool db1147 after a restart', diff saved to https://phabricator.wikimedia.org/P14109 and previous config saved to /var/cache/conftool/dbconfig/20210201-154618-root.json |
[production] |
| 15:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1147 (re)pooling @ 60%: Repool db1147 after a restart', diff saved to https://phabricator.wikimedia.org/P14108 and previous config saved to /var/cache/conftool/dbconfig/20210201-153115-root.json |
[production] |
| 15:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1147 (re)pooling @ 40%: Repool db1147 after a restart', diff saved to https://phabricator.wikimedia.org/P14107 and previous config saved to /var/cache/conftool/dbconfig/20210201-151611-root.json |
[production] |
| 15:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1147 (re)pooling @ 20%: Repool db1147 after a restart', diff saved to https://phabricator.wikimedia.org/P14106 and previous config saved to /var/cache/conftool/dbconfig/20210201-150107-root.json |
[production] |
| 14:53 |
<oblivian@cumin1001> |
conftool action : set/pooled=true; selector: dnsdisc=swift,name=codfw |
[production] |
| 14:53 |
<oblivian@cumin1001> |
conftool action : set/pooled=true; selector: dnsdisc=swift-ro,name=codfw |
[production] |
| 14:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1147 (re)pooling @ 10%: Repool db1147 after a restart', diff saved to https://phabricator.wikimedia.org/P14105 and previous config saved to /var/cache/conftool/dbconfig/20210201-144604-root.json |
[production] |
| 14:40 |
<marostegui> |
Restart mysql on db1147 T266483 |
[production] |
| 14:39 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1147 T266483', diff saved to https://phabricator.wikimedia.org/P14104 and previous config saved to /var/cache/conftool/dbconfig/20210201-143925-marostegui.json |
[production] |