|
2022-11-22
ยง
|
| 10:12 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on aux-k8s-etcd1002.eqiad.wmnet with reason: ganeti reboot |
[production] |
| 10:09 |
<godog> |
start backfilling data into graphite2004 - T315524 |
[production] |
| 10:05 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1200', diff saved to https://phabricator.wikimedia.org/P40499 and previous config saved to /var/cache/conftool/dbconfig/20221122-100515-ladsgroup.json |
[production] |
| 10:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1106', diff saved to https://phabricator.wikimedia.org/P40498 and previous config saved to /var/cache/conftool/dbconfig/20221122-100509-marostegui.json |
[production] |
| 10:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127', diff saved to https://phabricator.wikimedia.org/P40497 and previous config saved to /var/cache/conftool/dbconfig/20221122-100328-ladsgroup.json |
[production] |
| 10:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3315', diff saved to https://phabricator.wikimedia.org/P40496 and previous config saved to /var/cache/conftool/dbconfig/20221122-100323-marostegui.json |
[production] |
| 09:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2121 (T322618)', diff saved to https://phabricator.wikimedia.org/P40495 and previous config saved to /var/cache/conftool/dbconfig/20221122-095910-ladsgroup.json |
[production] |
| 09:58 |
<aborrero@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cloudvirt1050.eqiad.wmnet with OS bullseye |
[production] |
| 09:56 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2121 (T322618)', diff saved to https://phabricator.wikimedia.org/P40494 and previous config saved to /var/cache/conftool/dbconfig/20221122-095652-ladsgroup.json |
[production] |
| 09:56 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2121.codfw.wmnet with reason: Maintenance |
[production] |
| 09:56 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2121.codfw.wmnet with reason: Maintenance |
[production] |
| 09:56 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2120 (T322618)', diff saved to https://phabricator.wikimedia.org/P40493 and previous config saved to /var/cache/conftool/dbconfig/20221122-095631-ladsgroup.json |
[production] |
| 09:51 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1017.eqiad.wmnet |
[production] |
| 09:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1200 (T323214)', diff saved to https://phabricator.wikimedia.org/P40492 and previous config saved to /var/cache/conftool/dbconfig/20221122-095008-ladsgroup.json |
[production] |
| 09:50 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1106 (T321130)', diff saved to https://phabricator.wikimedia.org/P40491 and previous config saved to /var/cache/conftool/dbconfig/20221122-095003-marostegui.json |
[production] |
| 09:49 |
<oblivian@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/apertium: apply |
[production] |
| 09:48 |
<oblivian@deploy1002> |
helmfile [eqiad] START helmfile.d/services/apertium: apply |
[production] |
| 09:48 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127 (T322618)', diff saved to https://phabricator.wikimedia.org/P40490 and previous config saved to /var/cache/conftool/dbconfig/20221122-094821-ladsgroup.json |
[production] |
| 09:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3315 (T321126)', diff saved to https://phabricator.wikimedia.org/P40489 and previous config saved to /var/cache/conftool/dbconfig/20221122-094817-marostegui.json |
[production] |
| 09:47 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti1017.eqiad.wmnet |
[production] |
| 09:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1113:3315 (T321126)', diff saved to https://phabricator.wikimedia.org/P40488 and previous config saved to /var/cache/conftool/dbconfig/20221122-094645-marostegui.json |
[production] |
| 09:46 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:46 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110 (T321126)', diff saved to https://phabricator.wikimedia.org/P40487 and previous config saved to /var/cache/conftool/dbconfig/20221122-094635-marostegui.json |
[production] |
| 09:46 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1127 (T322618)', diff saved to https://phabricator.wikimedia.org/P40486 and previous config saved to /var/cache/conftool/dbconfig/20221122-094611-ladsgroup.json |
[production] |
| 09:46 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1127.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:45 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1127.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317 (T322618)', diff saved to https://phabricator.wikimedia.org/P40485 and previous config saved to /var/cache/conftool/dbconfig/20221122-094550-ladsgroup.json |
[production] |
| 09:41 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2120', diff saved to https://phabricator.wikimedia.org/P40484 and previous config saved to /var/cache/conftool/dbconfig/20221122-094125-ladsgroup.json |
[production] |
| 09:36 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on graphite2004.codfw.wmnet with reason: setup |
[production] |
| 09:36 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on graphite2004.codfw.wmnet with reason: setup |
[production] |
| 09:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1106 (T321130)', diff saved to https://phabricator.wikimedia.org/P40483 and previous config saved to /var/cache/conftool/dbconfig/20221122-093556-marostegui.json |
[production] |
| 09:35 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 10:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:35 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 10:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:35 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1106.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:35 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1106.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:31 |
<aborrero@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirt1050.eqiad.wmnet with reason: host reimage |
[production] |
| 09:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110', diff saved to https://phabricator.wikimedia.org/P40482 and previous config saved to /var/cache/conftool/dbconfig/20221122-093128-marostegui.json |
[production] |
| 09:30 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P40481 and previous config saved to /var/cache/conftool/dbconfig/20221122-093044-ladsgroup.json |
[production] |
| 09:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3311 (T321130)', diff saved to https://phabricator.wikimedia.org/P40480 and previous config saved to /var/cache/conftool/dbconfig/20221122-092845-marostegui.json |
[production] |
| 09:27 |
<aborrero@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirt1050.eqiad.wmnet with reason: host reimage |
[production] |
| 09:26 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2120', diff saved to https://phabricator.wikimedia.org/P40479 and previous config saved to /var/cache/conftool/dbconfig/20221122-092618-ladsgroup.json |
[production] |
| 09:25 |
<moritzm> |
failover Ganeti master in eqiad to ganeti1028 T311687 |
[production] |
| 09:24 |
<oblivian@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/apertium: apply |
[production] |
| 09:23 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/apertium: apply |
[production] |
| 09:22 |
<oblivian@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/apertium: apply |
[production] |
| 09:22 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/apertium: apply |
[production] |
| 09:19 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/apertium: apply |
[production] |
| 09:18 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/apertium: apply |
[production] |
| 09:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110', diff saved to https://phabricator.wikimedia.org/P40478 and previous config saved to /var/cache/conftool/dbconfig/20221122-091621-marostegui.json |
[production] |