|
2024-02-23
§
|
| 21:11 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1243', diff saved to https://phabricator.wikimedia.org/P57846 and previous config saved to /var/cache/conftool/dbconfig/20240223-211136-arnaudb.json |
[production] |
| 21:07 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db2196.codfw.wmnet with reason: host reimage |
[production] |
| 21:04 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2196.codfw.wmnet with reason: host reimage |
[production] |
| 20:56 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1243 (T357189)', diff saved to https://phabricator.wikimedia.org/P57845 and previous config saved to /var/cache/conftool/dbconfig/20240223-205630-arnaudb.json |
[production] |
| 20:48 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2196.codfw.wmnet with OS bookworm |
[production] |
| 20:42 |
<isaranto@deploy2002> |
helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'experimental' for release 'main' . |
[production] |
| 20:23 |
<RhinosF1> |
[relog due to stashbot errors] jhancock@cumin2002 ran cookbook SRE.hardware.upgrade-firmware for hosts db2201/db2204/db2197/db2198/db2202/db2203/db2205 and all END PASS |
[production] |
| 20:10 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['db2205'] |
[production] |
| 20:09 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['db2203'] |
[production] |
| 20:09 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['db2202'] |
[production] |
| 20:08 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['db2198'] |
[production] |
| 20:08 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['db2197'] |
[production] |
| 20:07 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['db2204'] |
[production] |
| 20:07 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['db2201'] |
[production] |
| 20:06 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['db2199'] |
[production] |
| 20:03 |
<eileen> |
civicrm upgraded from c50fcae3 to a65551c7 |
[production] |
| 20:00 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2205'] |
[production] |
| 20:00 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2204'] |
[production] |
| 19:59 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2203'] |
[production] |
| 19:59 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2202'] |
[production] |
| 19:59 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2201'] |
[production] |
| 19:59 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2199'] |
[production] |
| 19:59 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2198'] |
[production] |
| 19:59 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2197'] |
[production] |
| 19:58 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Depooling db1243 (T357189)', diff saved to https://phabricator.wikimedia.org/P57844 and previous config saved to /var/cache/conftool/dbconfig/20240223-195835-arnaudb.json |
[production] |
| 19:58 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1243.eqiad.wmnet with reason: Maintenance |
[production] |
| 19:58 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1243.eqiad.wmnet with reason: Maintenance |
[production] |
| 19:58 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1242 (T357189)', diff saved to https://phabricator.wikimedia.org/P57843 and previous config saved to /var/cache/conftool/dbconfig/20240223-195802-arnaudb.json |
[production] |
| 19:42 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1242', diff saved to https://phabricator.wikimedia.org/P57842 and previous config saved to /var/cache/conftool/dbconfig/20240223-194255-arnaudb.json |
[production] |
| 19:27 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1242', diff saved to https://phabricator.wikimedia.org/P57841 and previous config saved to /var/cache/conftool/dbconfig/20240223-192749-arnaudb.json |
[production] |
| 19:12 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1242 (T357189)', diff saved to https://phabricator.wikimedia.org/P57840 and previous config saved to /var/cache/conftool/dbconfig/20240223-191243-arnaudb.json |
[production] |
| 19:04 |
<eevans@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on restbase1036.eqiad.wmnet with reason: Bootstrapping — T354560 |
[production] |
| 19:04 |
<eevans@cumin1002> |
START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on restbase1036.eqiad.wmnet with reason: Bootstrapping — T354560 |
[production] |
| 19:03 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['db2220'] |
[production] |
| 19:03 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2220'] |
[production] |
| 19:03 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db2203.mgmt.codfw.wmnet with reboot policy FORCED |
[production] |
| 19:03 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['db2220'] |
[production] |
| 19:03 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2220'] |
[production] |
| 19:03 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['db2219'] |
[production] |
| 19:02 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2219'] |
[production] |
| 19:02 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['db2218'] |
[production] |
| 19:02 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2218'] |
[production] |
| 19:02 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['db2217'] |
[production] |
| 19:02 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2217'] |
[production] |
| 19:02 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['db2216'] |
[production] |
| 19:01 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2216'] |
[production] |
| 19:01 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['db2215'] |
[production] |
| 19:01 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db2215'] |
[production] |
| 18:58 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['db2200'] |
[production] |
| 18:57 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['db2214'] |
[production] |