|
2022-03-14
ยง
|
| 16:19 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2105.codfw.wmnet with reason: Maintenance |
[production] |
| 16:19 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112 (T298563)', diff saved to https://phabricator.wikimedia.org/P22440 and previous config saved to /var/cache/conftool/dbconfig/20220314-161943-marostegui.json |
[production] |
| 16:09 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1149', diff saved to https://phabricator.wikimedia.org/P22439 and previous config saved to /var/cache/conftool/dbconfig/20220314-160955-marostegui.json |
[production] |
| 16:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112', diff saved to https://phabricator.wikimedia.org/P22438 and previous config saved to /var/cache/conftool/dbconfig/20220314-160438-marostegui.json |
[production] |
| 15:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1149', diff saved to https://phabricator.wikimedia.org/P22437 and previous config saved to /var/cache/conftool/dbconfig/20220314-155450-marostegui.json |
[production] |
| 15:49 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112', diff saved to https://phabricator.wikimedia.org/P22436 and previous config saved to /var/cache/conftool/dbconfig/20220314-154933-marostegui.json |
[production] |
| 15:39 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1149 (T298294)', diff saved to https://phabricator.wikimedia.org/P22435 and previous config saved to /var/cache/conftool/dbconfig/20220314-153945-marostegui.json |
[production] |
| 15:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112 (T298563)', diff saved to https://phabricator.wikimedia.org/P22434 and previous config saved to /var/cache/conftool/dbconfig/20220314-153428-marostegui.json |
[production] |
| 15:10 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1149 (T298294)', diff saved to https://phabricator.wikimedia.org/P22432 and previous config saved to /var/cache/conftool/dbconfig/20220314-151025-marostegui.json |
[production] |
| 15:10 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1149.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:10 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1149.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:10 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148 (T298294)', diff saved to https://phabricator.wikimedia.org/P22431 and previous config saved to /var/cache/conftool/dbconfig/20220314-151017-marostegui.json |
[production] |
| 14:57 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cloudvirt1017.eqiad.wmnet with OS bullseye |
[production] |
| 14:55 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148', diff saved to https://phabricator.wikimedia.org/P22430 and previous config saved to /var/cache/conftool/dbconfig/20220314-145512-marostegui.json |
[production] |
| 14:55 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cloudvirt1016.eqiad.wmnet with OS bullseye |
[production] |
| 14:53 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1129 (T300775)', diff saved to https://phabricator.wikimedia.org/P22429 and previous config saved to /var/cache/conftool/dbconfig/20220314-145345-marostegui.json |
[production] |
| 14:53 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1129.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:53 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1129.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:51 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1112 (T298563)', diff saved to https://phabricator.wikimedia.org/P22428 and previous config saved to /var/cache/conftool/dbconfig/20220314-145109-marostegui.json |
[production] |
| 14:51 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:51 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:51 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1112.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:50 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1112.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148', diff saved to https://phabricator.wikimedia.org/P22427 and previous config saved to /var/cache/conftool/dbconfig/20220314-144007-marostegui.json |
[production] |
| 14:25 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148 (T298294)', diff saved to https://phabricator.wikimedia.org/P22426 and previous config saved to /var/cache/conftool/dbconfig/20220314-142502-marostegui.json |
[production] |
| 14:01 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=nginx,name=ms-fe1010.eqiad.wmnet |
[production] |
| 14:01 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=swift-fe,name=ms-fe1010.eqiad.wmnet |
[production] |
| 14:01 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=nginx,name=ms-fe1010.eqiad.wmnet |
[production] |
| 14:01 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=swift-fe,name=ms-fe1010.eqiad.wmnet |
[production] |
| 13:59 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirt1017.eqiad.wmnet with reason: host reimage |
[production] |
| 13:58 |
<herron> |
grafana1002:~# systemctl restart grafana-ldap-users-sync.service T303064 |
[production] |
| 13:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1148 (T298294)', diff saved to https://phabricator.wikimedia.org/P22425 and previous config saved to /var/cache/conftool/dbconfig/20220314-135744-marostegui.json |
[production] |
| 13:57 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1148.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:57 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1148.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1147 (T298294)', diff saved to https://phabricator.wikimedia.org/P22424 and previous config saved to /var/cache/conftool/dbconfig/20220314-135736-marostegui.json |
[production] |
| 13:57 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirt1016.eqiad.wmnet with reason: host reimage |
[production] |
| 13:57 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=nginx,name=ms-fe1011.eqiad.wmnet |
[production] |
| 13:57 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=swift-fe,name=ms-fe1011.eqiad.wmnet |
[production] |
| 13:56 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=nginx,name=ms-fe1011.eqiad.wmnet |
[production] |
| 13:56 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=swift-fe,name=ms-fe1011.eqiad.wmnet |
[production] |
| 13:55 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirt1017.eqiad.wmnet with reason: host reimage |
[production] |
| 13:54 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 13:53 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirt1016.eqiad.wmnet with reason: host reimage |
[production] |
| 13:52 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 13:52 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 13:50 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=nginx,name=ms-fe1012.eqiad.wmnet |
[production] |
| 13:50 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=swift-fe,name=ms-fe1012.eqiad.wmnet |
[production] |
| 13:50 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=nginx,name=ms-fe1012.eqiad.wmnet |
[production] |
| 13:49 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=swift-fe,name=ms-fe1012.eqiad.wmnet |
[production] |
| 13:48 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |