|
2022-03-30
ยง
|
| 20:07 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 20:06 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 20:06 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 20:05 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 20:02 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1143 (T298557)', diff saved to https://phabricator.wikimedia.org/P23889 and previous config saved to /var/cache/conftool/dbconfig/20220330-200236-marostegui.json |
[production] |
| 20:02 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1143.eqiad.wmnet with reason: Maintenance |
[production] |
| 20:02 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1143.eqiad.wmnet with reason: Maintenance |
[production] |
| 20:02 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1142 (T298557)', diff saved to https://phabricator.wikimedia.org/P23888 and previous config saved to /var/cache/conftool/dbconfig/20220330-200229-marostegui.json |
[production] |
| 20:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182', diff saved to https://phabricator.wikimedia.org/P23887 and previous config saved to /var/cache/conftool/dbconfig/20220330-200017-ladsgroup.json |
[production] |
| 19:56 |
<razzi@cumin1001> |
END (PASS) - Cookbook sre.kafka.reboot-workers (exit_code=0) for Kafka test-eqiad cluster: Reboot kafka nodes |
[production] |
| 19:47 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1142', diff saved to https://phabricator.wikimedia.org/P23886 and previous config saved to /var/cache/conftool/dbconfig/20220330-194723-marostegui.json |
[production] |
| 19:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182 (T298565)', diff saved to https://phabricator.wikimedia.org/P23885 and previous config saved to /var/cache/conftool/dbconfig/20220330-194512-ladsgroup.json |
[production] |
| 19:32 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1142', diff saved to https://phabricator.wikimedia.org/P23884 and previous config saved to /var/cache/conftool/dbconfig/20220330-193218-marostegui.json |
[production] |
| 19:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1098:3317 (T298565)', diff saved to https://phabricator.wikimedia.org/P23883 and previous config saved to /var/cache/conftool/dbconfig/20220330-192355-ladsgroup.json |
[production] |
| 19:23 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 19:23 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 19:23 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1158 (T298565)', diff saved to https://phabricator.wikimedia.org/P23882 and previous config saved to /var/cache/conftool/dbconfig/20220330-192347-ladsgroup.json |
[production] |
| 19:17 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1142 (T298557)', diff saved to https://phabricator.wikimedia.org/P23881 and previous config saved to /var/cache/conftool/dbconfig/20220330-191713-marostegui.json |
[production] |
| 19:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1158', diff saved to https://phabricator.wikimedia.org/P23880 and previous config saved to /var/cache/conftool/dbconfig/20220330-190842-ladsgroup.json |
[production] |
| 18:53 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1158', diff saved to https://phabricator.wikimedia.org/P23879 and previous config saved to /var/cache/conftool/dbconfig/20220330-185337-ladsgroup.json |
[production] |
| 18:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1182 (T298565)', diff saved to https://phabricator.wikimedia.org/P23878 and previous config saved to /var/cache/conftool/dbconfig/20220330-184458-ladsgroup.json |
[production] |
| 18:45 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1182.eqiad.wmnet with reason: Maintenance |
[production] |
| 18:44 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1182.eqiad.wmnet with reason: Maintenance |
[production] |
| 18:44 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 18:44 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 18:44 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3312 (T298565)', diff saved to https://phabricator.wikimedia.org/P23877 and previous config saved to /var/cache/conftool/dbconfig/20220330-184445-ladsgroup.json |
[production] |
| 18:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1158 (T298565)', diff saved to https://phabricator.wikimedia.org/P23876 and previous config saved to /var/cache/conftool/dbconfig/20220330-183832-ladsgroup.json |
[production] |
| 18:29 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3312', diff saved to https://phabricator.wikimedia.org/P23875 and previous config saved to /var/cache/conftool/dbconfig/20220330-182940-ladsgroup.json |
[production] |
| 18:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1158 (T298565)', diff saved to https://phabricator.wikimedia.org/P23874 and previous config saved to /var/cache/conftool/dbconfig/20220330-182537-ladsgroup.json |
[production] |
| 18:25 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on clouddb[1014,1018,1021].eqiad.wmnet,db1155.eqiad.wmnet with reason: Maintenance |
[production] |
| 18:25 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on clouddb[1014,1018,1021].eqiad.wmnet,db1155.eqiad.wmnet with reason: Maintenance |
[production] |
| 18:25 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1158.eqiad.wmnet with reason: Maintenance |
[production] |
| 18:25 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1158.eqiad.wmnet with reason: Maintenance |
[production] |
| 18:14 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3312', diff saved to https://phabricator.wikimedia.org/P23873 and previous config saved to /var/cache/conftool/dbconfig/20220330-181435-ladsgroup.json |
[production] |
| 18:11 |
<razzi@cumin1001> |
START - Cookbook sre.kafka.reboot-workers for Kafka test-eqiad cluster: Reboot kafka nodes |
[production] |
| 18:08 |
<razzi@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host zookeeper-test1002.eqiad.wmnet |
[production] |
| 18:03 |
<cmooney@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ms-be1069.eqiad.wmnet with reason: host reimage |
[production] |
| 18:01 |
<razzi@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host zookeeper-test1002.eqiad.wmnet |
[production] |
| 18:00 |
<razzi@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=99) for host zookeeper-test1002.eqiad.wmnet |
[production] |
| 18:00 |
<razzi@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host zookeeper-test1002.eqiad.wmnet |
[production] |
| 18:00 |
<cmooney@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be1069.eqiad.wmnet with reason: host reimage |
[production] |
| 17:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3312 (T298565)', diff saved to https://phabricator.wikimedia.org/P23872 and previous config saved to /var/cache/conftool/dbconfig/20220330-175930-ladsgroup.json |
[production] |
| 17:58 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1170:3312 (T298565)', diff saved to https://phabricator.wikimedia.org/P23871 and previous config saved to /var/cache/conftool/dbconfig/20220330-175822-ladsgroup.json |
[production] |
| 17:58 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:58 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:58 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1162 (T298565)', diff saved to https://phabricator.wikimedia.org/P23870 and previous config saved to /var/cache/conftool/dbconfig/20220330-175814-ladsgroup.json |
[production] |
| 17:47 |
<cmooney@cumin1001> |
START - Cookbook sre.hosts.reimage for host ms-be1069.eqiad.wmnet with OS stretch |
[production] |
| 17:46 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on 10 hosts with reason: Maintenance |
[production] |
| 17:46 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on 10 hosts with reason: Maintenance |
[production] |
| 17:46 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2121.codfw.wmnet with reason: Maintenance |
[production] |