|
2022-03-14
ยง
|
| 14:50 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1112.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148', diff saved to https://phabricator.wikimedia.org/P22427 and previous config saved to /var/cache/conftool/dbconfig/20220314-144007-marostegui.json |
[production] |
| 14:25 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148 (T298294)', diff saved to https://phabricator.wikimedia.org/P22426 and previous config saved to /var/cache/conftool/dbconfig/20220314-142502-marostegui.json |
[production] |
| 14:01 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=nginx,name=ms-fe1010.eqiad.wmnet |
[production] |
| 14:01 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=swift-fe,name=ms-fe1010.eqiad.wmnet |
[production] |
| 14:01 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=nginx,name=ms-fe1010.eqiad.wmnet |
[production] |
| 14:01 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=swift-fe,name=ms-fe1010.eqiad.wmnet |
[production] |
| 13:59 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirt1017.eqiad.wmnet with reason: host reimage |
[production] |
| 13:58 |
<herron> |
grafana1002:~# systemctl restart grafana-ldap-users-sync.service T303064 |
[production] |
| 13:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1148 (T298294)', diff saved to https://phabricator.wikimedia.org/P22425 and previous config saved to /var/cache/conftool/dbconfig/20220314-135744-marostegui.json |
[production] |
| 13:57 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1148.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:57 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1148.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1147 (T298294)', diff saved to https://phabricator.wikimedia.org/P22424 and previous config saved to /var/cache/conftool/dbconfig/20220314-135736-marostegui.json |
[production] |
| 13:57 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirt1016.eqiad.wmnet with reason: host reimage |
[production] |
| 13:57 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=nginx,name=ms-fe1011.eqiad.wmnet |
[production] |
| 13:57 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=swift-fe,name=ms-fe1011.eqiad.wmnet |
[production] |
| 13:56 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=nginx,name=ms-fe1011.eqiad.wmnet |
[production] |
| 13:56 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=swift-fe,name=ms-fe1011.eqiad.wmnet |
[production] |
| 13:55 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirt1017.eqiad.wmnet with reason: host reimage |
[production] |
| 13:54 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 13:53 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirt1016.eqiad.wmnet with reason: host reimage |
[production] |
| 13:52 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 13:52 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 13:50 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=nginx,name=ms-fe1012.eqiad.wmnet |
[production] |
| 13:50 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=swift-fe,name=ms-fe1012.eqiad.wmnet |
[production] |
| 13:50 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=nginx,name=ms-fe1012.eqiad.wmnet |
[production] |
| 13:49 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=swift-fe,name=ms-fe1012.eqiad.wmnet |
[production] |
| 13:48 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 13:45 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=nginx,name=ms-fe1009.eqiad.wmnet |
[production] |
| 13:45 |
<mvernon@cumin1001> |
conftool action : set/pooled=yes; selector: service=swift-fe,name=ms-fe1009.eqiad.wmnet |
[production] |
| 13:45 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=nginx,name=ms-fe1009.eqiad.wmnet |
[production] |
| 13:45 |
<mvernon@cumin1001> |
conftool action : set/weight=40; selector: service=swift-fe,name=ms-fe1009.eqiad.wmnet |
[production] |
| 13:44 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.reimage for host cloudvirt1017.eqiad.wmnet with OS bullseye |
[production] |
| 13:44 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:43 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1179 (T298563)', diff saved to https://phabricator.wikimedia.org/P22423 and previous config saved to /var/cache/conftool/dbconfig/20220314-134356-marostegui.json |
[production] |
| 13:43 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 13:43 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.reimage for host cloudvirt1016.eqiad.wmnet with OS bullseye |
[production] |
| 13:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1147', diff saved to https://phabricator.wikimedia.org/P22422 and previous config saved to /var/cache/conftool/dbconfig/20220314-134231-marostegui.json |
[production] |
| 13:39 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 13:39 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 13:36 |
<Emperor> |
restarting swift-proxy on ms-fe100[5-8] to update config to know about new eqiad frontends T303698 |
[production] |
| 13:34 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 13:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1179', diff saved to https://phabricator.wikimedia.org/P22421 and previous config saved to /var/cache/conftool/dbconfig/20220314-132849-marostegui.json |
[production] |
| 13:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1147', diff saved to https://phabricator.wikimedia.org/P22420 and previous config saved to /var/cache/conftool/dbconfig/20220314-132726-marostegui.json |
[production] |
| 13:25 |
<dcausse> |
restarting blazegraph on wdqs1006 (jvm stuck for 10hours) |
[production] |
| 13:24 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 13:23 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 13:23 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 13:22 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |