|
2024-04-09
ยง
|
| 15:20 |
<hnowlan@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-int: apply |
[production] |
| 15:20 |
<hnowlan@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-api-int: apply |
[production] |
| 15:20 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2105', diff saved to https://phabricator.wikimedia.org/P60095 and previous config saved to /var/cache/conftool/dbconfig/20240409-152005-arnaudb.json |
[production] |
| 15:17 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2125', diff saved to https://phabricator.wikimedia.org/P60094 and previous config saved to /var/cache/conftool/dbconfig/20240409-151750-arnaudb.json |
[production] |
| 15:16 |
<eoghan@cumin1002> |
START - Cookbook sre.gitlab.failover Failover of gitlab from gitlab1003.wikimedia.org to gitlab1004.wikimedia.org |
[production] |
| 15:06 |
<isaranto@deploy1002> |
helmfile [ml-staging-codfw] Ran 'sync' command on namespace 'experimental' for release 'main' . |
[production] |
| 15:04 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2105', diff saved to https://phabricator.wikimedia.org/P60093 and previous config saved to /var/cache/conftool/dbconfig/20240409-150458-arnaudb.json |
[production] |
| 15:02 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2125', diff saved to https://phabricator.wikimedia.org/P60092 and previous config saved to /var/cache/conftool/dbconfig/20240409-150242-arnaudb.json |
[production] |
| 15:02 |
<sukhe@cumin1002> |
START - Cookbook sre.hosts.reimage for host cp4052.ulsfo.wmnet with OS bullseye |
[production] |
| 15:01 |
<vgutierrez@cumin1002> |
START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-ulsfo and not P{cp[4037,4041,4045,4049,4052].ulsfo.wmnet} and A:cp |
[production] |
| 14:49 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2105 (T360332)', diff saved to https://phabricator.wikimedia.org/P60091 and previous config saved to /var/cache/conftool/dbconfig/20240409-144950-arnaudb.json |
[production] |
| 14:47 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2125 (T360332)', diff saved to https://phabricator.wikimedia.org/P60090 and previous config saved to /var/cache/conftool/dbconfig/20240409-144735-arnaudb.json |
[production] |
| 14:44 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Depooling db2125 (T360332)', diff saved to https://phabricator.wikimedia.org/P60089 and previous config saved to /var/cache/conftool/dbconfig/20240409-144447-arnaudb.json |
[production] |
| 14:44 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2125.codfw.wmnet with reason: Maintenance |
[production] |
| 14:44 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2125.codfw.wmnet with reason: Maintenance |
[production] |
| 14:44 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2107 (T360332)', diff saved to https://phabricator.wikimedia.org/P60088 and previous config saved to /var/cache/conftool/dbconfig/20240409-144424-arnaudb.json |
[production] |
| 14:35 |
<sukhe@cumin1002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts cp4052.ulsfo.wmnet |
[production] |
| 14:34 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Depooling db2105 (T360332)', diff saved to https://phabricator.wikimedia.org/P60087 and previous config saved to /var/cache/conftool/dbconfig/20240409-143445-arnaudb.json |
[production] |
| 14:34 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2105.codfw.wmnet with reason: Maintenance |
[production] |
| 14:34 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2105.codfw.wmnet with reason: Maintenance |
[production] |
| 14:33 |
<sukhe@cumin1002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts cp4052.ulsfo.wmnet |
[production] |
| 14:33 |
<btullis@cumin1002> |
END (FAIL) - Cookbook sre.ganeti.makevm (exit_code=99) for new host matomo1003.eqiad.wmnet |
[production] |
| 14:33 |
<btullis@cumin1002> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host matomo1003.eqiad.wmnet with OS bookworm |
[production] |
| 14:30 |
<sukhe@cumin1002> |
END (ERROR) - Cookbook sre.hardware.upgrade-firmware (exit_code=97) upgrade firmware for hosts cp4052.ulsfo.wmnet |
[production] |
| 14:30 |
<sukhe@cumin1002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts cp4052.ulsfo.wmnet |
[production] |
| 14:29 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:29 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:29 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2107', diff saved to https://phabricator.wikimedia.org/P60086 and previous config saved to /var/cache/conftool/dbconfig/20240409-142916-arnaudb.json |
[production] |
| 14:28 |
<sukhe@cumin1002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cp4052.ulsfo.wmnet with OS bullseye |
[production] |
| 14:24 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1240.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:24 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1240.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:24 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1223 (T360332)', diff saved to https://phabricator.wikimedia.org/P60085 and previous config saved to /var/cache/conftool/dbconfig/20240409-142414-arnaudb.json |
[production] |
| 14:18 |
<elukey@cumin1002> |
END (PASS) - Cookbook sre.cassandra.roll-restart (exit_code=0) for nodes matching A:aqs-eqiad: Deploy new Truststore - elukey@cumin1002 |
[production] |
| 14:14 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2107', diff saved to https://phabricator.wikimedia.org/P60084 and previous config saved to /var/cache/conftool/dbconfig/20240409-141410-arnaudb.json |
[production] |
| 14:09 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1223', diff saved to https://phabricator.wikimedia.org/P60083 and previous config saved to /var/cache/conftool/dbconfig/20240409-140906-arnaudb.json |
[production] |
| 14:07 |
<sukhe@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cp4052.ulsfo.wmnet with reason: host reimage |
[production] |
| 14:07 |
<btullis@cumin1002> |
START - Cookbook sre.hosts.reimage for host dumpsdata1005.eqiad.wmnet with OS bullseye |
[production] |
| 14:04 |
<sukhe@cumin1002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cp4052.ulsfo.wmnet with reason: host reimage |
[production] |
| 14:03 |
<cgoubert@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply |
[production] |
| 14:02 |
<cgoubert@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-debug: apply |
[production] |
| 13:59 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2107 (T360332)', diff saved to https://phabricator.wikimedia.org/P60082 and previous config saved to /var/cache/conftool/dbconfig/20240409-135902-arnaudb.json |
[production] |
| 13:57 |
<btullis@cumin1002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host dumpsdata1004.eqiad.wmnet with OS bullseye |
[production] |
| 13:57 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.mysql.clone (exit_code=0) Will create a clone of db2112.codfw.wmnet onto db2212.codfw.wmnet |
[production] |
| 13:56 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Depooling db2107 (T360332)', diff saved to https://phabricator.wikimedia.org/P60081 and previous config saved to /var/cache/conftool/dbconfig/20240409-135621-arnaudb.json |
[production] |
| 13:56 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2107.codfw.wmnet with reason: Maintenance |
[production] |
| 13:56 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2107.codfw.wmnet with reason: Maintenance |
[production] |
| 13:55 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2097.codfw.wmnet with reason: Maintenance |
[production] |
| 13:55 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2097.codfw.wmnet with reason: Maintenance |
[production] |
| 13:54 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:54 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |