|
2022-10-21
ยง
|
| 13:35 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2163.codfw.wmnet with reason: Maintenance |
[production] |
| 13:35 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2163.codfw.wmnet with reason: Maintenance |
[production] |
| 13:35 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2162 (T321312)', diff saved to https://phabricator.wikimedia.org/P35871 and previous config saved to /var/cache/conftool/dbconfig/20221021-133509-ladsgroup.json |
[production] |
| 13:34 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be1041.eqiad.wmnet |
[production] |
| 13:34 |
<cgoubert@cumin1001> |
conftool action : set/pooled=yes; selector: dc=eqiad,cluster=kubernetes,service=kubesvc,name=kubernetes1005.eqiad.wmnet |
[production] |
| 13:33 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2042.codfw.wmnet |
[production] |
| 13:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1104 (T321312)', diff saved to https://phabricator.wikimedia.org/P35870 and previous config saved to /var/cache/conftool/dbconfig/20221021-133231-ladsgroup.json |
[production] |
| 13:32 |
<claime> |
kubernetes1005:~$ sudo systemctl reset-failed ifup@ens13.service T273026 |
[production] |
| 13:31 |
<bking@cumin2002> |
START - Cookbook sre.elasticsearch.rolling-operation Operation.REBOOT (1 nodes at a time) for ElasticSearch cluster cloudelastic: apply updates - bking@cumin2002 - T321310 |
[production] |
| 13:28 |
<btullis@cumin1001> |
START - Cookbook sre.k8s.reboot-nodes rolling reboot on A:dse-k8s-worker |
[production] |
| 13:27 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be1041.eqiad.wmnet |
[production] |
| 13:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1104 (T321312)', diff saved to https://phabricator.wikimedia.org/P35869 and previous config saved to /var/cache/conftool/dbconfig/20221021-132716-ladsgroup.json |
[production] |
| 13:27 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1104.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:27 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2042.codfw.wmnet |
[production] |
| 13:26 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1104.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:26 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3318 (T321312)', diff saved to https://phabricator.wikimedia.org/P35868 and previous config saved to /var/cache/conftool/dbconfig/20221021-132652-ladsgroup.json |
[production] |
| 13:25 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2041.codfw.wmnet |
[production] |
| 13:20 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2162', diff saved to https://phabricator.wikimedia.org/P35867 and previous config saved to /var/cache/conftool/dbconfig/20221021-132003-ladsgroup.json |
[production] |
| 13:19 |
<sukhe@cumin2002> |
END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host cp4047.ulsfo.wmnet with OS buster |
[production] |
| 13:18 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2041.codfw.wmnet |
[production] |
| 13:17 |
<cgoubert@cumin1001> |
START - Cookbook sre.k8s.reboot-nodes rolling reboot on A:wikikube-worker-eqiad |
[production] |
| 13:16 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2040.codfw.wmnet |
[production] |
| 13:11 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3318', diff saved to https://phabricator.wikimedia.org/P35866 and previous config saved to /var/cache/conftool/dbconfig/20221021-131145-ladsgroup.json |
[production] |
| 13:09 |
<mvernon@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2040.codfw.wmnet |
[production] |
| 13:07 |
<sukhe@cumin2002> |
START - Cookbook sre.hosts.reimage for host cp4047.ulsfo.wmnet with OS buster |
[production] |
| 13:04 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2162', diff saved to https://phabricator.wikimedia.org/P35865 and previous config saved to /var/cache/conftool/dbconfig/20221021-130456-ladsgroup.json |
[production] |
| 13:00 |
<cgoubert@cumin1001> |
END (FAIL) - Cookbook sre.k8s.reboot-nodes (exit_code=1) rolling reboot on A:wikikube-worker-codfw |
[production] |
| 12:56 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3318', diff saved to https://phabricator.wikimedia.org/P35864 and previous config saved to /var/cache/conftool/dbconfig/20221021-125639-ladsgroup.json |
[production] |
| 12:55 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes2024.codfw.wmnet |
[production] |
| 12:49 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2162 (T321312)', diff saved to https://phabricator.wikimedia.org/P35863 and previous config saved to /var/cache/conftool/dbconfig/20221021-124950-ladsgroup.json |
[production] |
| 12:48 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes2024.codfw.wmnet |
[production] |
| 12:47 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes2023.codfw.wmnet |
[production] |
| 12:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2162 (T321312)', diff saved to https://phabricator.wikimedia.org/P35862 and previous config saved to /var/cache/conftool/dbconfig/20221021-124327-ladsgroup.json |
[production] |
| 12:43 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2162.codfw.wmnet with reason: Maintenance |
[production] |
| 12:43 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2162.codfw.wmnet with reason: Maintenance |
[production] |
| 12:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2161 (T321312)', diff saved to https://phabricator.wikimedia.org/P35861 and previous config saved to /var/cache/conftool/dbconfig/20221021-124302-ladsgroup.json |
[production] |
| 12:42 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes2023.codfw.wmnet |
[production] |
| 12:41 |
<dcausse> |
restarting blazegraph on wdqs1013 (BlazegraphFreeAllocatorsDecreasingRapidly) |
[production] |
| 12:41 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3318 (T321312)', diff saved to https://phabricator.wikimedia.org/P35860 and previous config saved to /var/cache/conftool/dbconfig/20221021-124132-ladsgroup.json |
[production] |
| 12:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317 (T321312)', diff saved to https://phabricator.wikimedia.org/P35859 and previous config saved to /var/cache/conftool/dbconfig/20221021-123815-ladsgroup.json |
[production] |
| 12:35 |
<claime> |
rebooted kubernetes2006.codfw.wmnet manually - root cause T273026 |
[production] |
| 12:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2161', diff saved to https://phabricator.wikimedia.org/P35858 and previous config saved to /var/cache/conftool/dbconfig/20221021-122755-ladsgroup.json |
[production] |
| 12:23 |
<cgoubert@cumin1001> |
conftool action : set/pooled=yes; selector: dc=codfw,cluster=kubernetes,service=kubesvc,name=kubernetes2006.codfw.wmnet |
[production] |
| 12:23 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P35857 and previous config saved to /var/cache/conftool/dbconfig/20221021-122308-ladsgroup.json |
[production] |
| 12:12 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2161', diff saved to https://phabricator.wikimedia.org/P35856 and previous config saved to /var/cache/conftool/dbconfig/20221021-121249-ladsgroup.json |
[production] |
| 12:11 |
<mvernon@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-cluster (exit_code=0) |
[production] |
| 12:09 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes2006.codfw.wmnet |
[production] |
| 12:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P35855 and previous config saved to /var/cache/conftool/dbconfig/20221021-120802-ladsgroup.json |
[production] |
| 12:03 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes2006.codfw.wmnet |
[production] |
| 11:57 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2161 (T321312)', diff saved to https://phabricator.wikimedia.org/P35854 and previous config saved to /var/cache/conftool/dbconfig/20221021-115742-ladsgroup.json |
[production] |