|
2022-01-17
ยง
|
| 14:58 |
<btullis@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM an-airflow1003.eqiad.wmnet |
[production] |
| 14:50 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db2132.codfw.wmnet with OS bullseye |
[production] |
| 14:50 |
<btullis@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM an-airflow1002.eqiad.wmnet |
[production] |
| 14:48 |
<btullis@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM an-airflow1002.eqiad.wmnet |
[production] |
| 14:44 |
<moritzm> |
imported cassandra 3.11.11 to component/cassandradev for stretch-wikimedia and buster-wikimedia T298805 |
[production] |
| 14:41 |
<moritzm> |
systemctl reset-failed ifup@ens5.service on an-airflow1001 T273026 |
[production] |
| 14:39 |
<btullis@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM an-airflow1001.eqiad.wmnet |
[production] |
| 14:37 |
<hnowlan> |
removing restbase2009 from cassandra configs |
[production] |
| 14:30 |
<btullis@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM an-airflow1001.eqiad.wmnet |
[production] |
| 14:16 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.reimage for host db2132.codfw.wmnet with OS bullseye |
[production] |
| 14:15 |
<marostegui> |
Reimage db2132 to Bullseye T299344 |
[production] |
| 13:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Remove recentchanges group from s3 eqiad T263127', diff saved to https://phabricator.wikimedia.org/P18762 and previous config saved to /var/cache/conftool/dbconfig/20220117-134520-marostegui.json |
[production] |
| 12:49 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db1151.eqiad.wmnet with OS bullseye |
[production] |
| 12:19 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.reimage for host db1151.eqiad.wmnet with OS bullseye |
[production] |
| 12:14 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db2142.codfw.wmnet with OS bullseye |
[production] |
| 11:40 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.reimage for host db2142.codfw.wmnet with OS bullseye |
[production] |
| 11:30 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM kafkamon1002.eqiad.wmnet |
[production] |
| 11:26 |
<jmm@cumin2002> |
START - Cookbook sre.ganeti.reboot-vm for VM kafkamon1002.eqiad.wmnet |
[production] |
| 11:08 |
<moritzm> |
switching kubetcd1006 to DRBD-backed storage (required for ganeti update) |
[production] |
| 11:03 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on kubetcd1006.eqiad.wmnet with reason: switch to drbd storage |
[production] |
| 11:03 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on kubetcd1006.eqiad.wmnet with reason: switch to drbd storage |
[production] |
| 11:00 |
<moritzm> |
systemctl reset-failed ifup@ens5.service on kubetcd1005 T273026 |
[production] |
| 10:56 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM ml-serve-ctrl1002.eqiad.wmnet |
[production] |
| 10:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Remove recentchangeslinked group from s3 eqiad T263127', diff saved to https://phabricator.wikimedia.org/P18761 and previous config saved to /var/cache/conftool/dbconfig/20220117-104801-marostegui.json |
[production] |
| 10:47 |
<elukey@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM ml-serve-ctrl1002.eqiad.wmnet |
[production] |
| 10:45 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db1152.eqiad.wmnet with OS bullseye |
[production] |
| 10:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1131 (T285149)', diff saved to https://phabricator.wikimedia.org/P18760 and previous config saved to /var/cache/conftool/dbconfig/20220117-104459-marostegui.json |
[production] |
| 10:44 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM ml-serve-ctrl1001.eqiad.wmnet |
[production] |
| 10:42 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db1153.eqiad.wmnet with OS bullseye |
[production] |
| 10:42 |
<elukey@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM ml-serve-ctrl1001.eqiad.wmnet |
[production] |
| 10:32 |
<moritzm> |
switching kubetcd1005 to DRBD-backed storage (required for ganeti update) |
[production] |
| 10:31 |
<jayme@deploy1002> |
helmfile [staging] DONE helmfile.d/services/wikifeeds: sync on staging |
[production] |
| 10:31 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on kubetcd1005.eqiad.wmnet with reason: switch to drbd storage |
[production] |
| 10:31 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on kubetcd1005.eqiad.wmnet with reason: switch to drbd storage |
[production] |
| 10:30 |
<jayme@deploy1002> |
helmfile [staging] DONE helmfile.d/services/wikifeeds: apply on production |
[production] |
| 10:30 |
<jayme@deploy1002> |
helmfile [staging] START helmfile.d/services/wikifeeds: apply on staging |
[production] |
| 10:29 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1131', diff saved to https://phabricator.wikimedia.org/P18759 and previous config saved to /var/cache/conftool/dbconfig/20220117-102954-marostegui.json |
[production] |
| 10:17 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.reimage for host db1152.eqiad.wmnet with OS bullseye |
[production] |
| 10:15 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.reimage for host db1153.eqiad.wmnet with OS bullseye |
[production] |
| 10:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1131', diff saved to https://phabricator.wikimedia.org/P18758 and previous config saved to /var/cache/conftool/dbconfig/20220117-101450-marostegui.json |
[production] |
| 10:06 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db2144.codfw.wmnet with OS bullseye |
[production] |
| 10:04 |
<moritzm> |
switching kubetcd1004 to DRBD-backed storage (required for ganeti update) |
[production] |
| 10:03 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on kubetcd1004.eqiad.wmnet with reason: switch to drbd storage |
[production] |
| 10:03 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on kubetcd1004.eqiad.wmnet with reason: switch to drbd storage |
[production] |
| 10:02 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db2143.codfw.wmnet with OS bullseye |
[production] |
| 09:59 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1131 (T285149)', diff saved to https://phabricator.wikimedia.org/P18757 and previous config saved to /var/cache/conftool/dbconfig/20220117-095945-marostegui.json |
[production] |
| 09:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1131 (T285149)', diff saved to https://phabricator.wikimedia.org/P18756 and previous config saved to /var/cache/conftool/dbconfig/20220117-095837-marostegui.json |
[production] |
| 09:58 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1131.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:58 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1131.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3316 (T285149)', diff saved to https://phabricator.wikimedia.org/P18755 and previous config saved to /var/cache/conftool/dbconfig/20220117-095830-marostegui.json |
[production] |