|
2023-08-28
ยง
|
| 11:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2145', diff saved to https://phabricator.wikimedia.org/P51587 and previous config saved to /var/cache/conftool/dbconfig/20230828-113455-ladsgroup.json |
[production] |
| 11:33 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti2018.codfw.wmnet |
[production] |
| 11:31 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1190', diff saved to https://phabricator.wikimedia.org/P51585 and previous config saved to /var/cache/conftool/dbconfig/20230828-113136-ladsgroup.json |
[production] |
| 11:30 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2172', diff saved to https://phabricator.wikimedia.org/P51584 and previous config saved to /var/cache/conftool/dbconfig/20230828-113050-ladsgroup.json |
[production] |
| 11:29 |
<jmm@cumin2002> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti2018.codfw.wmnet |
[production] |
| 11:23 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti2017.codfw.wmnet |
[production] |
| 11:23 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti2017.codfw.wmnet |
[production] |
| 11:22 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134', diff saved to https://phabricator.wikimedia.org/P51583 and previous config saved to /var/cache/conftool/dbconfig/20230828-112206-ladsgroup.json |
[production] |
| 11:20 |
<fabfur> |
enable puppet and start pybal on lvs4009 (T344587) |
[production] |
| 11:19 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2145 (T343718)', diff saved to https://phabricator.wikimedia.org/P51582 and previous config saved to /var/cache/conftool/dbconfig/20230828-111949-ladsgroup.json |
[production] |
| 11:18 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host lvs4009.ulsfo.wmnet |
[production] |
| 11:16 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1190', diff saved to https://phabricator.wikimedia.org/P51581 and previous config saved to /var/cache/conftool/dbconfig/20230828-111630-ladsgroup.json |
[production] |
| 11:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2172', diff saved to https://phabricator.wikimedia.org/P51580 and previous config saved to /var/cache/conftool/dbconfig/20230828-111544-ladsgroup.json |
[production] |
| 11:15 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti2017.codfw.wmnet |
[production] |
| 11:15 |
<fabfur@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host lvs4009.ulsfo.wmnet |
[production] |
| 11:15 |
<kamila@deploy1002> |
Finished scap: base image update due to T344991 (duration: 09m 31s) |
[production] |
| 11:11 |
<moritzm> |
bounce ferm on ml-serve10001 |
[production] |
| 11:07 |
<jmm@cumin2002> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti2017.codfw.wmnet |
[production] |
| 11:07 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134', diff saved to https://phabricator.wikimedia.org/P51579 and previous config saved to /var/cache/conftool/dbconfig/20230828-110700-ladsgroup.json |
[production] |
| 11:05 |
<kamila@deploy1002> |
Started scap: base image update due to T344991 |
[production] |
| 11:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1190 (T344589)', diff saved to https://phabricator.wikimedia.org/P51578 and previous config saved to /var/cache/conftool/dbconfig/20230828-110124-ladsgroup.json |
[production] |
| 11:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2172 (T344589)', diff saved to https://phabricator.wikimedia.org/P51577 and previous config saved to /var/cache/conftool/dbconfig/20230828-110038-ladsgroup.json |
[production] |
| 10:58 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on restbase1027.eqiad.wmnet with reason: T345058 - service probes flapping |
[production] |
| 10:57 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on restbase1027.eqiad.wmnet with reason: T345058 - service probes flapping |
[production] |
| 10:57 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti2016.codfw.wmnet |
[production] |
| 10:57 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti2016.codfw.wmnet |
[production] |
| 10:55 |
<fabfur> |
disable puppet and stop pybal on lvs4009 for reboot (T344587) |
[production] |
| 10:55 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1190 (T344589)', diff saved to https://phabricator.wikimedia.org/P51576 and previous config saved to /var/cache/conftool/dbconfig/20230828-105503-ladsgroup.json |
[production] |
| 10:54 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1190.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:54 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1190.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2172 (T344589)', diff saved to https://phabricator.wikimedia.org/P51575 and previous config saved to /var/cache/conftool/dbconfig/20230828-105407-ladsgroup.json |
[production] |
| 10:54 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2172.codfw.wmnet with reason: Maintenance |
[production] |
| 10:53 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2172.codfw.wmnet with reason: Maintenance |
[production] |
| 10:53 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2155 (T344589)', diff saved to https://phabricator.wikimedia.org/P51574 and previous config saved to /var/cache/conftool/dbconfig/20230828-105342-ladsgroup.json |
[production] |
| 10:51 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134 (T343718)', diff saved to https://phabricator.wikimedia.org/P51573 and previous config saved to /var/cache/conftool/dbconfig/20230828-105153-ladsgroup.json |
[production] |
| 10:50 |
<moritzm> |
installing exim4 bugfix updates from Bookworm point release |
[production] |
| 10:50 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1150.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:50 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1150.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1149 (T344589)', diff saved to https://phabricator.wikimedia.org/P51572 and previous config saved to /var/cache/conftool/dbconfig/20230828-105002-ladsgroup.json |
[production] |
| 10:49 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti2016.codfw.wmnet |
[production] |
| 10:46 |
<jmm@cumin2002> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti2016.codfw.wmnet |
[production] |
| 10:46 |
<jelto@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/miscweb: apply |
[production] |
| 10:44 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2145 (T343718)', diff saved to https://phabricator.wikimedia.org/P51571 and previous config saved to /var/cache/conftool/dbconfig/20230828-104407-ladsgroup.json |
[production] |
| 10:44 |
<jelto@deploy1002> |
helmfile [eqiad] START helmfile.d/services/miscweb: apply |
[production] |
| 10:44 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2145.codfw.wmnet with reason: Maintenance |
[production] |
| 10:43 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2145.codfw.wmnet with reason: Maintenance |
[production] |
| 10:42 |
<cgoubert@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-ext: apply |
[production] |
| 10:42 |
<cgoubert@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-api-ext: apply |
[production] |
| 10:41 |
<cgoubert@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mw-api-ext: apply |
[production] |
| 10:41 |
<cgoubert@deploy1002> |
helmfile [codfw] START helmfile.d/services/mw-api-ext: apply |
[production] |