|
2021-03-23
ยง
|
| 09:45 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 09:45 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 09:45 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1014.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:44 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 09:44 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 09:43 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1014.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:43 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1013.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Pool db1165 into s6 with minimal weight T258361', diff saved to https://phabricator.wikimedia.org/P15028 and previous config saved to /var/cache/conftool/dbconfig/20210323-094257-marostegui.json |
[production] |
| 09:41 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1012.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:41 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1016.eqiad.wmnet |
[production] |
| 09:41 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1013.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:40 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1015.eqiad.wmnet |
[production] |
| 09:40 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1005.eqiad.wmnet |
[production] |
| 09:39 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1011.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:38 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1012.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:37 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1010.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:36 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1011.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:36 |
<jayme@cumin1001> |
END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) for 2:00:00 on kubernetes1004.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:35 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1009.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:34 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1010.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:33 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes1017.eqiad.wmnet |
[production] |
| 09:33 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1008.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:32 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1009.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:32 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Add db1165 to dbctl, depooled - T258361', diff saved to https://phabricator.wikimedia.org/P15027 and previous config saved to /var/cache/conftool/dbconfig/20210323-093246-marostegui.json |
[production] |
| 09:31 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1007.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:31 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1004.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:30 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1008.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:29 |
<jayme@cumin1001> |
END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) for 2:00:00 on kubernetes1003.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:29 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1002.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:28 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1007.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:28 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1003.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:27 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1002.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:26 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1001.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:26 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1086 to clone db1181 T275633', diff saved to https://phabricator.wikimedia.org/P15025 and previous config saved to /var/cache/conftool/dbconfig/20210323-092600-marostegui.json |
[production] |
| 09:24 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1001.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:18 |
<akosiaris@cumin1001> |
conftool action : set/pooled=true; selector: dc=eqiad,cluster=kubernetes,name=kubernetes1017.eqiad.wmnet |
[production] |
| 09:17 |
<akosiaris@cumin1001> |
conftool action : set/pooled=yes; selector: dc=eqiad,service=kubemaster,cluster=kubernetes |
[production] |
| 09:17 |
<akosiaris@cumin1001> |
conftool action : set/weight=10; selector: dc=eqiad,service=kubemaster,cluster=kubernetes |
[production] |
| 09:16 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1017.eqiad.wmnet |
[production] |
| 09:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1147', diff saved to https://phabricator.wikimedia.org/P15024 and previous config saved to /var/cache/conftool/dbconfig/20210323-091432-marostegui.json |
[production] |
| 09:05 |
<akosiaris> |
reboot kubetcd100[456] for kernel upgrades. T277741 T273278 |
[production] |
| 09:04 |
<akosiaris> |
empty etcd T277741 |
[production] |
| 08:43 |
<akosiaris> |
poweroff argon and chlorine T277741 |
[production] |
| 08:41 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1098:3317 (re)pooling @ 100%: Slowly repool db1098:3317', diff saved to https://phabricator.wikimedia.org/P15023 and previous config saved to /var/cache/conftool/dbconfig/20210323-083957-root.json |
[production] |
| 08:41 |
<akosiaris@cumin1001> |
conftool action : set/pooled=false; selector: name=eqiad,dnsdisc=zotero |
[production] |
| 08:41 |
<akosiaris@cumin1001> |
conftool action : set/pooled=false; selector: name=eqiad,dnsdisc=wikifeeds |
[production] |
| 08:41 |
<akosiaris@cumin1001> |
conftool action : set/pooled=false; selector: name=eqiad,dnsdisc=termbox |
[production] |
| 08:41 |
<akosiaris@cumin1001> |
conftool action : set/pooled=false; selector: name=eqiad,dnsdisc=similar-users |
[production] |
| 08:41 |
<akosiaris@cumin1001> |
conftool action : set/pooled=false; selector: name=eqiad,dnsdisc=sessionstore |
[production] |
| 08:40 |
<akosiaris@cumin1001> |
conftool action : set/pooled=false; selector: name=eqiad,dnsdisc=recommendation-api |
[production] |