|
2022-10-24
ยง
|
| 09:18 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes2015.codfw.wmnet |
[production] |
| 09:18 |
<oblivian@deploy1002> |
Started scap: Backport for [[gerrit:839499|Stop assigning the PHP_ENGINE cookie (T271736)]] |
[production] |
| 09:18 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2141.codfw.wmnet with reason: Maintenance |
[production] |
| 09:18 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130 (T321312)', diff saved to https://phabricator.wikimedia.org/P35993 and previous config saved to /var/cache/conftool/dbconfig/20221024-091801-ladsgroup.json |
[production] |
| 09:17 |
<claime> |
kubernetes2015:~$ sudo systemctl reset-failed ifup@ens13.service T273026 |
[production] |
| 09:15 |
<mvernon@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=1) for host ms-be2051.codfw.wmnet |
[production] |
| 09:12 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes2015.codfw.wmnet |
[production] |
| 09:11 |
<cgoubert@cumin1001> |
conftool action : set/pooled=no; selector: dc=codfw,cluster=kubernetes,service=kubesvc,name=kubernetes2015.codfw.wmnet |
[production] |
| 09:10 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be1047.eqiad.wmnet |
[production] |
| 09:09 |
<claime> |
Starting october reboots of lingering wikikube codfw hosts |
[production] |
| 09:08 |
<cgoubert@cumin1001> |
conftool action : set/pooled=yes; selector: dc=eqiad,cluster=kubernetes,service=kubemaster,name=kubemaster1002.eqiad.wmnet |
[production] |
| 09:08 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubemaster1002.eqiad.wmnet |
[production] |
| 09:03 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be1047.eqiad.wmnet |
[production] |
| 09:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130', diff saved to https://phabricator.wikimedia.org/P35991 and previous config saved to /var/cache/conftool/dbconfig/20221024-090255-ladsgroup.json |
[production] |
| 09:01 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubemaster1002.eqiad.wmnet |
[production] |
| 09:01 |
<cgoubert@cumin1001> |
conftool action : set/pooled=no; selector: dc=eqiad,cluster=kubernetes,service=kubemaster,name=kubemaster1002.eqiad.wmnet |
[production] |
| 09:01 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be1046.eqiad.wmnet |
[production] |
| 09:01 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2051.codfw.wmnet |
[production] |
| 09:00 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2050.codfw.wmnet |
[production] |
| 09:00 |
<cgoubert@cumin1001> |
conftool action : set/pooled=yes; selector: dc=eqiad,cluster=kubernetes,service=kubemaster,name=kubemaster1001.eqiad.wmnet |
[production] |
| 08:59 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubemaster1001.eqiad.wmnet |
[production] |
| 08:53 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubemaster1001.eqiad.wmnet |
[production] |
| 08:52 |
<cgoubert@cumin1001> |
conftool action : set/pooled=no; selector: dc=eqiad,cluster=kubernetes,service=kubemaster,name=kubemaster1001.eqiad.wmnet |
[production] |
| 08:52 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2050.codfw.wmnet |
[production] |
| 08:51 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2049.codfw.wmnet |
[production] |
| 08:48 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be1046.eqiad.wmnet |
[production] |
| 08:47 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130', diff saved to https://phabricator.wikimedia.org/P35990 and previous config saved to /var/cache/conftool/dbconfig/20221024-084748-ladsgroup.json |
[production] |
| 08:45 |
<cgoubert@cumin1001> |
conftool action : set/pooled=yes; selector: dc=eqiad,cluster=kubernetes,service=kubesvc,name=kubernetes1016.eqiad.wmnet |
[production] |
| 08:44 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes1016.eqiad.wmnet |
[production] |
| 08:44 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'configure' for AS: 199524 |
[production] |
| 08:44 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be1045.eqiad.wmnet |
[production] |
| 08:42 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.peering with action 'configure' for AS: 199524 |
[production] |
| 08:40 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1106 (T321312)', diff saved to https://phabricator.wikimedia.org/P35989 and previous config saved to /var/cache/conftool/dbconfig/20221024-084037-ladsgroup.json |
[production] |
| 08:40 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:40 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:40 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1106.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:40 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1106.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:39 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 31042 |
[production] |
| 08:39 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312 (T321312)', diff saved to https://phabricator.wikimedia.org/P35988 and previous config saved to /var/cache/conftool/dbconfig/20221024-083955-ladsgroup.json |
[production] |
| 08:39 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1016.eqiad.wmnet |
[production] |
| 08:39 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.peering with action 'email' for AS: 31042 |
[production] |
| 08:38 |
<cgoubert@cumin1001> |
conftool action : set/pooled=no; selector: dc=eqiad,cluster=kubernetes,service=kubesvc,name=kubernetes1016.eqiad.wmnet |
[production] |
| 08:38 |
<cgoubert@cumin1001> |
conftool action : set/pooled=yes; selector: dc=eqiad,cluster=kubernetes,service=kubesvc,name=kubernetes1015.eqiad.wmnet |
[production] |
| 08:37 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2049.codfw.wmnet |
[production] |
| 08:37 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes1015.eqiad.wmnet |
[production] |
| 08:37 |
<claime> |
kubernetes1015:~$ sudo systemctl reset-failed ifup@ens13.service T273026 |
[production] |
| 08:36 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be1045.eqiad.wmnet |
[production] |
| 08:35 |
<Emperor> |
set thanos ring replicas to 3.50 T311690 |
[production] |
| 08:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130 (T321312)', diff saved to https://phabricator.wikimedia.org/P35987 and previous config saved to /var/cache/conftool/dbconfig/20221024-083242-ladsgroup.json |
[production] |
| 08:30 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1015.eqiad.wmnet |
[production] |