|
2022-11-22
ยง
|
| 09:48 |
<oblivian@deploy1002> |
helmfile [eqiad] START helmfile.d/services/apertium: apply |
[production] |
| 09:48 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127 (T322618)', diff saved to https://phabricator.wikimedia.org/P40490 and previous config saved to /var/cache/conftool/dbconfig/20221122-094821-ladsgroup.json |
[production] |
| 09:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3315 (T321126)', diff saved to https://phabricator.wikimedia.org/P40489 and previous config saved to /var/cache/conftool/dbconfig/20221122-094817-marostegui.json |
[production] |
| 09:47 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti1017.eqiad.wmnet |
[production] |
| 09:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1113:3315 (T321126)', diff saved to https://phabricator.wikimedia.org/P40488 and previous config saved to /var/cache/conftool/dbconfig/20221122-094645-marostegui.json |
[production] |
| 09:46 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:46 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110 (T321126)', diff saved to https://phabricator.wikimedia.org/P40487 and previous config saved to /var/cache/conftool/dbconfig/20221122-094635-marostegui.json |
[production] |
| 09:46 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1127 (T322618)', diff saved to https://phabricator.wikimedia.org/P40486 and previous config saved to /var/cache/conftool/dbconfig/20221122-094611-ladsgroup.json |
[production] |
| 09:46 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1127.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:45 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1127.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317 (T322618)', diff saved to https://phabricator.wikimedia.org/P40485 and previous config saved to /var/cache/conftool/dbconfig/20221122-094550-ladsgroup.json |
[production] |
| 09:41 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2120', diff saved to https://phabricator.wikimedia.org/P40484 and previous config saved to /var/cache/conftool/dbconfig/20221122-094125-ladsgroup.json |
[production] |
| 09:36 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on graphite2004.codfw.wmnet with reason: setup |
[production] |
| 09:36 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on graphite2004.codfw.wmnet with reason: setup |
[production] |
| 09:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1106 (T321130)', diff saved to https://phabricator.wikimedia.org/P40483 and previous config saved to /var/cache/conftool/dbconfig/20221122-093556-marostegui.json |
[production] |
| 09:35 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 10:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:35 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 10:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:35 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1106.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:35 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1106.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:31 |
<aborrero@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirt1050.eqiad.wmnet with reason: host reimage |
[production] |
| 09:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110', diff saved to https://phabricator.wikimedia.org/P40482 and previous config saved to /var/cache/conftool/dbconfig/20221122-093128-marostegui.json |
[production] |
| 09:30 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P40481 and previous config saved to /var/cache/conftool/dbconfig/20221122-093044-ladsgroup.json |
[production] |
| 09:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3311 (T321130)', diff saved to https://phabricator.wikimedia.org/P40480 and previous config saved to /var/cache/conftool/dbconfig/20221122-092845-marostegui.json |
[production] |
| 09:27 |
<aborrero@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirt1050.eqiad.wmnet with reason: host reimage |
[production] |
| 09:26 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2120', diff saved to https://phabricator.wikimedia.org/P40479 and previous config saved to /var/cache/conftool/dbconfig/20221122-092618-ladsgroup.json |
[production] |
| 09:25 |
<moritzm> |
failover Ganeti master in eqiad to ganeti1028 T311687 |
[production] |
| 09:24 |
<oblivian@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/apertium: apply |
[production] |
| 09:23 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/apertium: apply |
[production] |
| 09:22 |
<oblivian@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/apertium: apply |
[production] |
| 09:22 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/apertium: apply |
[production] |
| 09:19 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/apertium: apply |
[production] |
| 09:18 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/apertium: apply |
[production] |
| 09:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110', diff saved to https://phabricator.wikimedia.org/P40478 and previous config saved to /var/cache/conftool/dbconfig/20221122-091621-marostegui.json |
[production] |
| 09:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P40477 and previous config saved to /var/cache/conftool/dbconfig/20221122-091537-ladsgroup.json |
[production] |
| 09:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3311', diff saved to https://phabricator.wikimedia.org/P40476 and previous config saved to /var/cache/conftool/dbconfig/20221122-091339-marostegui.json |
[production] |
| 09:13 |
<aborrero@cumin1001> |
START - Cookbook sre.hosts.reimage for host cloudvirt1050.eqiad.wmnet with OS bullseye |
[production] |
| 09:11 |
<jmm@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=1) for host cumin2002.codfw.wmnet |
[production] |
| 09:11 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2120 (T322618)', diff saved to https://phabricator.wikimedia.org/P40475 and previous config saved to /var/cache/conftool/dbconfig/20221122-091112-ladsgroup.json |
[production] |
| 09:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2120 (T322618)', diff saved to https://phabricator.wikimedia.org/P40474 and previous config saved to /var/cache/conftool/dbconfig/20221122-090854-ladsgroup.json |
[production] |
| 09:08 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2120.codfw.wmnet with reason: Maintenance |
[production] |
| 09:08 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2120.codfw.wmnet with reason: Maintenance |
[production] |
| 09:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2108 (T322618)', diff saved to https://phabricator.wikimedia.org/P40473 and previous config saved to /var/cache/conftool/dbconfig/20221122-090833-ladsgroup.json |
[production] |
| 09:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110 (T321126)', diff saved to https://phabricator.wikimedia.org/P40472 and previous config saved to /var/cache/conftool/dbconfig/20221122-090115-marostegui.json |
[production] |
| 09:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317 (T322618)', diff saved to https://phabricator.wikimedia.org/P40471 and previous config saved to /var/cache/conftool/dbconfig/20221122-090030-ladsgroup.json |
[production] |
| 09:00 |
<jmm@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cumin2002.codfw.wmnet |
[production] |
| 08:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1110 (T321126)', diff saved to https://phabricator.wikimedia.org/P40470 and previous config saved to /var/cache/conftool/dbconfig/20221122-085843-marostegui.json |
[production] |
| 08:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3311', diff saved to https://phabricator.wikimedia.org/P40469 and previous config saved to /var/cache/conftool/dbconfig/20221122-085832-marostegui.json |
[production] |
| 08:58 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1110.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:58 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1110.eqiad.wmnet with reason: Maintenance |
[production] |