|
2022-03-29
ยง
|
| 17:47 |
<moritzm> |
installing tiff security updates |
[production] |
| 17:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3316 (T297189)', diff saved to https://phabricator.wikimedia.org/P23590 and previous config saved to /var/cache/conftool/dbconfig/20220329-174518-marostegui.json |
[production] |
| 17:29 |
<mutante> |
gitlab2001 - systemctl reset-failed |
[production] |
| 17:23 |
<mutante> |
gitlab2001 - did not come back from reboot via cookbook. logged in via console. then "s/ens5/ens13" in /etc/network/interfaces ; reboot ; issue was like T272555 and others |
[production] |
| 17:22 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1171.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:22 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1171.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:13 |
<razzi@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on an-tool1005.eqiad.wmnet with reason: Testing deploy of superset 1.4.2 to staging |
[production] |
| 17:13 |
<razzi@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on an-tool1005.eqiad.wmnet with reason: Testing deploy of superset 1.4.2 to staging |
[production] |
| 17:11 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ores2003.codfw.wmnet |
[production] |
| 17:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1129 (T298565)', diff saved to https://phabricator.wikimedia.org/P23589 and previous config saved to /var/cache/conftool/dbconfig/20220329-170924-ladsgroup.json |
[production] |
| 17:09 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1129.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:09 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1129.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312 (T298565)', diff saved to https://phabricator.wikimedia.org/P23588 and previous config saved to /var/cache/conftool/dbconfig/20220329-170916-ladsgroup.json |
[production] |
| 17:04 |
<klausman@cumin2002> |
END (ERROR) - Cookbook sre.hosts.reboot-single (exit_code=97) for host ml-staging2001.codfw.wmnet |
[production] |
| 17:04 |
<klausman@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ml-staging2001.codfw.wmnet |
[production] |
| 17:03 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ores2003.codfw.wmnet |
[production] |
| 17:03 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ores2002.codfw.wmnet |
[production] |
| 17:00 |
<bking@cumin1001> |
START - Cookbook sre.wdqs.reboot |
[production] |
| 17:00 |
<aokoth@cumin1001> |
END (FAIL) - Cookbook sre.ganeti.reboot-vm (exit_code=99) for VM gitlab2001.wikimedia.org |
[production] |
| 16:55 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ores2002.codfw.wmnet |
[production] |
| 16:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312', diff saved to https://phabricator.wikimedia.org/P23586 and previous config saved to /var/cache/conftool/dbconfig/20220329-165411-ladsgroup.json |
[production] |
| 16:51 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ores2001.codfw.wmnet |
[production] |
| 16:45 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host ores2001.codfw.wmnet |
[production] |
| 16:39 |
<hashar@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/blubberoid: apply |
[production] |
| 16:39 |
<hashar@deploy1002> |
helmfile [codfw] START helmfile.d/services/blubberoid: apply |
[production] |
| 16:39 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312', diff saved to https://phabricator.wikimedia.org/P23584 and previous config saved to /var/cache/conftool/dbconfig/20220329-163906-ladsgroup.json |
[production] |
| 16:39 |
<hashar@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/blubberoid: apply |
[production] |
| 16:38 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 16:38 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 16:38 |
<hashar@deploy1002> |
helmfile [eqiad] START helmfile.d/services/blubberoid: apply |
[production] |
| 16:38 |
<hashar@deploy1002> |
helmfile [staging] DONE helmfile.d/services/blubberoid: apply |
[production] |
| 16:37 |
<hashar@deploy1002> |
helmfile [staging] START helmfile.d/services/blubberoid: apply |
[production] |
| 16:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1098:3316 (T297189)', diff saved to https://phabricator.wikimedia.org/P23583 and previous config saved to /var/cache/conftool/dbconfig/20220329-163503-marostegui.json |
[production] |
| 16:35 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 16:35 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 16:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T297189)', diff saved to https://phabricator.wikimedia.org/P23582 and previous config saved to /var/cache/conftool/dbconfig/20220329-163455-marostegui.json |
[production] |
| 16:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312 (T298565)', diff saved to https://phabricator.wikimedia.org/P23581 and previous config saved to /var/cache/conftool/dbconfig/20220329-162401-ladsgroup.json |
[production] |
| 16:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1105:3312 (T298565)', diff saved to https://phabricator.wikimedia.org/P23580 and previous config saved to /var/cache/conftool/dbconfig/20220329-162153-ladsgroup.json |
[production] |
| 16:21 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 16:21 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 16:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182 (T298565)', diff saved to https://phabricator.wikimedia.org/P23579 and previous config saved to /var/cache/conftool/dbconfig/20220329-162146-ladsgroup.json |
[production] |
| 16:21 |
<bking@cumin1001> |
END (FAIL) - Cookbook sre.wdqs.reboot (exit_code=99) |
[production] |
| 16:19 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P23578 and previous config saved to /var/cache/conftool/dbconfig/20220329-161950-marostegui.json |
[production] |
| 16:19 |
<aokoth@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM gitlab2001.wikimedia.org |
[production] |
| 16:17 |
<aokoth@cumin1001> |
END (FAIL) - Cookbook sre.ganeti.reboot-vm (exit_code=99) for VM gitlab2001.wikimedia.org |
[production] |
| 16:17 |
<aokoth@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM gitlab2001.wikimedia.org |
[production] |
| 16:06 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182', diff saved to https://phabricator.wikimedia.org/P23577 and previous config saved to /var/cache/conftool/dbconfig/20220329-160640-ladsgroup.json |
[production] |
| 16:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P23576 and previous config saved to /var/cache/conftool/dbconfig/20220329-160446-marostegui.json |
[production] |
| 15:54 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on 10 hosts with reason: Maintenance |
[production] |
| 15:54 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on 10 hosts with reason: Maintenance |
[production] |