|
2022-11-22
ยง
|
| 14:45 |
<pt1979@cumin2002> |
END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host dbprov1004.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 14:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1132 (T321130)', diff saved to https://phabricator.wikimedia.org/P40617 and previous config saved to /var/cache/conftool/dbconfig/20221122-144519-marostegui.json |
[production] |
| 14:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2128 (T321126)', diff saved to https://phabricator.wikimedia.org/P40616 and previous config saved to /var/cache/conftool/dbconfig/20221122-144507-marostegui.json |
[production] |
| 14:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2182 (T322618)', diff saved to https://phabricator.wikimedia.org/P40615 and previous config saved to /var/cache/conftool/dbconfig/20221122-144458-ladsgroup.json |
[production] |
| 14:45 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 10:00:00 on db2094.codfw.wmnet with reason: Maintenance |
[production] |
| 14:45 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 10:00:00 on db2094.codfw.wmnet with reason: Maintenance |
[production] |
| 14:45 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2182.codfw.wmnet with reason: Maintenance |
[production] |
| 14:44 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db2128.codfw.wmnet with reason: Maintenance |
[production] |
| 14:44 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db2128.codfw.wmnet with reason: Maintenance |
[production] |
| 14:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2123 (T321126)', diff saved to https://phabricator.wikimedia.org/P40614 and previous config saved to /var/cache/conftool/dbconfig/20221122-144446-marostegui.json |
[production] |
| 14:44 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2182.codfw.wmnet with reason: Maintenance |
[production] |
| 14:44 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317 (T322618)', diff saved to https://phabricator.wikimedia.org/P40613 and previous config saved to /var/cache/conftool/dbconfig/20221122-144436-ladsgroup.json |
[production] |
| 14:43 |
<bking@cumin2002> |
START - Cookbook sre.elasticsearch.rolling-operation Operation.RESTART (3 nodes at a time) for ElasticSearch cluster search_eqiad: apply config changes - bking@cumin2002 - T319020 |
[production] |
| 14:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1194 (T322618)', diff saved to https://phabricator.wikimedia.org/P40612 and previous config saved to /var/cache/conftool/dbconfig/20221122-144232-ladsgroup.json |
[production] |
| 14:41 |
<sukhe@cumin2002> |
START - Cookbook sre.hosts.reimage for host lvs4009.ulsfo.wmnet with OS buster |
[production] |
| 14:41 |
<oblivian@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/api-gateway: apply |
[production] |
| 14:41 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:41 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:40 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:40 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:40 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1194 (T322618)', diff saved to https://phabricator.wikimedia.org/P40611 and previous config saved to /var/cache/conftool/dbconfig/20221122-144023-ladsgroup.json |
[production] |
| 14:40 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1194.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:40 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1194.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:40 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1191 (T322618)', diff saved to https://phabricator.wikimedia.org/P40610 and previous config saved to /var/cache/conftool/dbconfig/20221122-144002-ladsgroup.json |
[production] |
| 14:39 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:39 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:39 |
<oblivian@deploy1002> |
helmfile [eqiad] START helmfile.d/services/api-gateway: apply |
[production] |
| 14:39 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:39 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:38 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:38 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:35 |
<oblivian@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/api-gateway: apply |
[production] |
| 14:34 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/api-gateway: apply |
[production] |
| 14:33 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/api-gateway: apply |
[production] |
| 14:32 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1132 (T321130)', diff saved to https://phabricator.wikimedia.org/P40609 and previous config saved to /var/cache/conftool/dbconfig/20221122-143224-marostegui.json |
[production] |
| 14:32 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/api-gateway: apply |
[production] |
| 14:32 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1132.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:32 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1132.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:32 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1128 (T321130)', diff saved to https://phabricator.wikimedia.org/P40608 and previous config saved to /var/cache/conftool/dbconfig/20221122-143203-marostegui.json |
[production] |
| 14:29 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2123', diff saved to https://phabricator.wikimedia.org/P40607 and previous config saved to /var/cache/conftool/dbconfig/20221122-142939-marostegui.json |
[production] |
| 14:29 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317', diff saved to https://phabricator.wikimedia.org/P40606 and previous config saved to /var/cache/conftool/dbconfig/20221122-142930-ladsgroup.json |
[production] |
| 14:28 |
<btullis@cumin1001> |
START - Cookbook sre.wikireplicas.add-wiki |
[production] |
| 14:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1191', diff saved to https://phabricator.wikimedia.org/P40605 and previous config saved to /var/cache/conftool/dbconfig/20221122-142455-ladsgroup.json |
[production] |
| 14:20 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1032.eqiad.wmnet |
[production] |
| 14:18 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host dbprov1004.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 14:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1128', diff saved to https://phabricator.wikimedia.org/P40604 and previous config saved to /var/cache/conftool/dbconfig/20221122-141656-marostegui.json |
[production] |
| 14:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2123', diff saved to https://phabricator.wikimedia.org/P40603 and previous config saved to /var/cache/conftool/dbconfig/20221122-141433-marostegui.json |
[production] |
| 14:14 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317', diff saved to https://phabricator.wikimedia.org/P40602 and previous config saved to /var/cache/conftool/dbconfig/20221122-141423-ladsgroup.json |
[production] |
| 14:13 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti1032.eqiad.wmnet |
[production] |
| 14:13 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on kubestagetcd1004.eqiad.wmnet with reason: ganeti reboot |
[production] |