|
2023-05-09
ยง
|
| 12:02 |
<kart_> |
Updated cxserver to 2023-05-08-134152-production (T336115, T335987, T331835) |
[production] |
| 11:58 |
<eoghan@cumin1001> |
START - Cookbook sre.hosts.decommission for hosts aphlict1001.eqiad.wmnet |
[production] |
| 11:58 |
<kartik@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/cxserver: apply |
[production] |
| 11:57 |
<kartik@deploy1002> |
helmfile [eqiad] START helmfile.d/services/cxserver: apply |
[production] |
| 11:55 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130', diff saved to https://phabricator.wikimedia.org/P47987 and previous config saved to /var/cache/conftool/dbconfig/20230509-115547-ladsgroup.json |
[production] |
| 11:53 |
<kartik@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/cxserver: apply |
[production] |
| 11:53 |
<kartik@deploy1002> |
helmfile [codfw] START helmfile.d/services/cxserver: apply |
[production] |
| 11:49 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1166', diff saved to https://phabricator.wikimedia.org/P47986 and previous config saved to /var/cache/conftool/dbconfig/20230509-114903-ladsgroup.json |
[production] |
| 11:45 |
<kartik@deploy1002> |
helmfile [staging] DONE helmfile.d/services/cxserver: apply |
[production] |
| 11:45 |
<kartik@deploy1002> |
helmfile [staging] START helmfile.d/services/cxserver: apply |
[production] |
| 11:40 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130', diff saved to https://phabricator.wikimedia.org/P47985 and previous config saved to /var/cache/conftool/dbconfig/20230509-114041-ladsgroup.json |
[production] |
| 11:36 |
<kart_> |
Updated MinT to 2023-05-09-110213-production (T331505, T335725, T331505) |
[production] |
| 11:33 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1166', diff saved to https://phabricator.wikimedia.org/P47984 and previous config saved to /var/cache/conftool/dbconfig/20230509-113357-ladsgroup.json |
[production] |
| 11:33 |
<kartik@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/machinetranslation: apply |
[production] |
| 11:31 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ganeti.reimage (exit_code=0) for host ldap-rw1001.wikimedia.org with OS bullseye |
[production] |
| 11:29 |
<kartik@deploy1002> |
helmfile [eqiad] START helmfile.d/services/machinetranslation: apply |
[production] |
| 11:27 |
<kartik@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/machinetranslation: apply |
[production] |
| 11:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130 (T335845)', diff saved to https://phabricator.wikimedia.org/P47983 and previous config saved to /var/cache/conftool/dbconfig/20230509-112535-ladsgroup.json |
[production] |
| 11:23 |
<kartik@deploy1002> |
helmfile [codfw] START helmfile.d/services/machinetranslation: apply |
[production] |
| 11:20 |
<kartik@deploy1002> |
helmfile [staging] DONE helmfile.d/services/machinetranslation: apply |
[production] |
| 11:20 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ldap-rw1001.wikimedia.org with reason: host reimage |
[production] |
| 11:18 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1166 (T335845)', diff saved to https://phabricator.wikimedia.org/P47982 and previous config saved to /var/cache/conftool/dbconfig/20230509-111851-ladsgroup.json |
[production] |
| 11:18 |
<kartik@deploy1002> |
helmfile [staging] START helmfile.d/services/machinetranslation: apply |
[production] |
| 11:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2130 (T335845)', diff saved to https://phabricator.wikimedia.org/P47981 and previous config saved to /var/cache/conftool/dbconfig/20230509-111755-ladsgroup.json |
[production] |
| 11:17 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2130.codfw.wmnet with reason: Maintenance |
[production] |
| 11:17 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2130.codfw.wmnet with reason: Maintenance |
[production] |
| 11:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116 (T335845)', diff saved to https://phabricator.wikimedia.org/P47980 and previous config saved to /var/cache/conftool/dbconfig/20230509-111730-ladsgroup.json |
[production] |
| 11:16 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on ldap-rw1001.wikimedia.org with reason: host reimage |
[production] |
| 11:12 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1166 (T335845)', diff saved to https://phabricator.wikimedia.org/P47979 and previous config saved to /var/cache/conftool/dbconfig/20230509-111235-ladsgroup.json |
[production] |
| 11:12 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1166.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:12 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1166.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:12 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1123 (T335845)', diff saved to https://phabricator.wikimedia.org/P47978 and previous config saved to /var/cache/conftool/dbconfig/20230509-111211-ladsgroup.json |
[production] |
| 11:08 |
<jmm@cumin2002> |
START - Cookbook sre.ganeti.reimage for host ldap-rw1001.wikimedia.org with OS bullseye |
[production] |
| 11:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116', diff saved to https://phabricator.wikimedia.org/P47977 and previous config saved to /var/cache/conftool/dbconfig/20230509-110222-ladsgroup.json |
[production] |
| 10:57 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1123', diff saved to https://phabricator.wikimedia.org/P47976 and previous config saved to /var/cache/conftool/dbconfig/20230509-105704-ladsgroup.json |
[production] |
| 10:47 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116', diff saved to https://phabricator.wikimedia.org/P47975 and previous config saved to /var/cache/conftool/dbconfig/20230509-104715-ladsgroup.json |
[production] |
| 10:45 |
<aborrero@cumin2002> |
END (FAIL) - Cookbook sre.hosts.decommission (exit_code=1) for hosts cloudcontrol2001-dev.wikimedia.org |
[production] |
| 10:45 |
<aborrero@cumin2002> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 10:44 |
<aborrero@cumin2002> |
START - Cookbook sre.dns.netbox |
[production] |
| 10:42 |
<volans@cumin1001> |
END (PASS) - Cookbook sre.netbox.update-extras (exit_code=0) rolling update on A:netbox |
[production] |
| 10:41 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1123', diff saved to https://phabricator.wikimedia.org/P47974 and previous config saved to /var/cache/conftool/dbconfig/20230509-104158-ladsgroup.json |
[production] |
| 10:39 |
<aborrero@cumin2002> |
START - Cookbook sre.hosts.decommission for hosts cloudcontrol2001-dev.wikimedia.org |
[production] |
| 10:36 |
<volans@cumin1001> |
START - Cookbook sre.netbox.update-extras rolling update on A:netbox |
[production] |
| 10:36 |
<volans@cumin1001> |
END (PASS) - Cookbook sre.netbox.update-extras (exit_code=0) rolling update on A:netbox |
[production] |
| 10:32 |
<aborrero@cumin2002> |
END (ERROR) - Cookbook sre.hosts.decommission (exit_code=97) for hosts cloudcontrol2001-dev.wikimedia.org |
[production] |
| 10:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116 (T335845)', diff saved to https://phabricator.wikimedia.org/P47973 and previous config saved to /var/cache/conftool/dbconfig/20230509-103209-ladsgroup.json |
[production] |
| 10:29 |
<aborrero@cumin2002> |
START - Cookbook sre.hosts.decommission for hosts cloudcontrol2001-dev.wikimedia.org |
[production] |
| 10:26 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1123 (T335845)', diff saved to https://phabricator.wikimedia.org/P47972 and previous config saved to /var/cache/conftool/dbconfig/20230509-102652-ladsgroup.json |
[production] |
| 10:26 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2116 (T335845)', diff saved to https://phabricator.wikimedia.org/P47971 and previous config saved to /var/cache/conftool/dbconfig/20230509-102644-ladsgroup.json |
[production] |
| 10:26 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2116.codfw.wmnet with reason: Maintenance |
[production] |