|
2023-11-21
ยง
|
| 11:37 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on kubernetes2041.codfw.wmnet with reason: NIC 1 Port 1 network link is down |
[production] |
| 11:35 |
<jmm@cumin2002> |
START - Cookbook sre.puppet.migrate-host for host titan2002.codfw.wmnet |
[production] |
| 11:22 |
<mvernon@cumin2002> |
START - Cookbook sre.hosts.reimage for host ms-fe2014.codfw.wmnet with OS bullseye |
[production] |
| 11:21 |
<jayme@deploy1002> |
helmfile [staging-eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 11:21 |
<jayme@deploy1002> |
helmfile [staging-eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 11:20 |
<Emperor> |
depool ms-fe2014 to reimage with new envoy TLS setup T317616 |
[production] |
| 11:13 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.puppet.migrate-host (exit_code=0) for host mwlog2002.codfw.wmnet |
[production] |
| 11:05 |
<jmm@cumin2002> |
START - Cookbook sre.puppet.migrate-host for host mwlog2002.codfw.wmnet |
[production] |
| 11:00 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.puppet.migrate-role (exit_code=0) for role: gitlab_runner |
[production] |
| 10:50 |
<jmm@cumin2002> |
START - Cookbook sre.puppet.migrate-role for role: gitlab_runner |
[production] |
| 10:35 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.puppet.migrate-host (exit_code=0) for host gerrit2002.wikimedia.org |
[production] |
| 10:35 |
<jbond> |
upload new wmf-certificates packages |
[production] |
| 10:25 |
<jelto@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host gitlab-runner1003.eqiad.wmnet |
[production] |
| 10:22 |
<jmm@cumin2002> |
START - Cookbook sre.puppet.migrate-host for host gerrit2002.wikimedia.org |
[production] |
| 10:21 |
<hnowlan@deploy1002> |
helmfile [staging] DONE helmfile.d/services/api-gateway: apply |
[production] |
| 10:18 |
<jelto@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host gitlab-runner1003.eqiad.wmnet |
[production] |
| 10:11 |
<hnowlan@deploy1002> |
helmfile [staging] START helmfile.d/services/api-gateway: apply |
[production] |
| 10:10 |
<hnowlan@deploy1002> |
helmfile [staging] DONE helmfile.d/services/api-gateway: apply |
[production] |
| 10:10 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.puppet.migrate-host (exit_code=0) for host gitlab-runner1002.eqiad.wmnet |
[production] |
| 10:06 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2180 (re)pooling @ 100%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53670 and previous config saved to /var/cache/conftool/dbconfig/20231121-100607-arnaudb.json |
[production] |
| 10:05 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2178 (re)pooling @ 100%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53669 and previous config saved to /var/cache/conftool/dbconfig/20231121-100536-arnaudb.json |
[production] |
| 10:03 |
<oblivian@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 10:02 |
<oblivian@deploy2002> |
helmfile [codfw] START helmfile.d/services/mobileapps: apply |
[production] |
| 10:02 |
<oblivian@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 10:01 |
<oblivian@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mobileapps: apply |
[production] |
| 10:00 |
<jmm@cumin2002> |
START - Cookbook sre.puppet.migrate-host for host gitlab-runner1002.eqiad.wmnet |
[production] |
| 10:00 |
<hnowlan@deploy1002> |
helmfile [staging] START helmfile.d/services/api-gateway: apply |
[production] |
| 09:53 |
<oblivian@deploy2002> |
helmfile [staging] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 09:51 |
<oblivian@deploy2002> |
helmfile [staging] START helmfile.d/services/mobileapps: apply |
[production] |
| 09:51 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2180 (re)pooling @ 90%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53667 and previous config saved to /var/cache/conftool/dbconfig/20231121-095102-arnaudb.json |
[production] |
| 09:50 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2178 (re)pooling @ 90%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53666 and previous config saved to /var/cache/conftool/dbconfig/20231121-095031-arnaudb.json |
[production] |
| 09:35 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2180 (re)pooling @ 75%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53665 and previous config saved to /var/cache/conftool/dbconfig/20231121-093557-arnaudb.json |
[production] |
| 09:35 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2178 (re)pooling @ 75%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53664 and previous config saved to /var/cache/conftool/dbconfig/20231121-093526-arnaudb.json |
[production] |
| 09:24 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.loadbalancer.restart-pybal (exit_code=0) rolling-restart of pybal on P{lvs[2011-2013].codfw.wmnet} and A:lvs (T351069) |
[production] |
| 09:20 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2180 (re)pooling @ 60%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53663 and previous config saved to /var/cache/conftool/dbconfig/20231121-092052-arnaudb.json |
[production] |
| 09:20 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2178 (re)pooling @ 60%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53662 and previous config saved to /var/cache/conftool/dbconfig/20231121-092021-arnaudb.json |
[production] |
| 09:19 |
<vgutierrez@cumin1001> |
START - Cookbook sre.loadbalancer.restart-pybal rolling-restart of pybal on P{lvs[2011-2013].codfw.wmnet} and A:lvs (T351069) |
[production] |
| 09:18 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.loadbalancer.restart-pybal (exit_code=0) rolling-restart of pybal on P{lvs2014.codfw.wmnet} and A:lvs (T351069) |
[production] |
| 09:18 |
<vgutierrez@cumin1001> |
START - Cookbook sre.loadbalancer.restart-pybal rolling-restart of pybal on P{lvs2014.codfw.wmnet} and A:lvs (T351069) |
[production] |
| 09:17 |
<vgutierrez> |
updating pybal to 1.5.14 on codfw - T351069 |
[production] |
| 09:17 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.loadbalancer.restart-pybal (exit_code=0) rolling-restart of pybal on P{lvs[5004-5005].eqsin.wmnet} and A:lvs (T351069) |
[production] |
| 09:16 |
<vgutierrez@cumin1001> |
START - Cookbook sre.loadbalancer.restart-pybal rolling-restart of pybal on P{lvs[5004-5005].eqsin.wmnet} and A:lvs (T351069) |
[production] |
| 09:15 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.loadbalancer.restart-pybal (exit_code=0) rolling-restart of pybal on P{lvs5006.eqsin.wmnet} and A:lvs (T351069) |
[production] |
| 09:15 |
<vgutierrez@cumin1001> |
START - Cookbook sre.loadbalancer.restart-pybal rolling-restart of pybal on P{lvs5006.eqsin.wmnet} and A:lvs (T351069) |
[production] |
| 09:14 |
<vgutierrez> |
updating pybal to 1.5.14 on eqsin - T351069 |
[production] |
| 09:12 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.loadbalancer.restart-pybal (exit_code=0) rolling-restart of pybal on P{lvs[4008-4009].ulsfo.wmnet} and A:lvs (T351069) |
[production] |
| 09:10 |
<vgutierrez> |
updating pybal to 1.5.14 on ulsfo - T351069 |
[production] |
| 09:09 |
<vgutierrez@cumin1001> |
START - Cookbook sre.loadbalancer.restart-pybal rolling-restart of pybal on P{lvs[4008-4009].ulsfo.wmnet} and A:lvs (T351069) |
[production] |
| 09:05 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2180 (re)pooling @ 45%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53661 and previous config saved to /var/cache/conftool/dbconfig/20231121-090547-arnaudb.json |
[production] |
| 09:05 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'db2178 (re)pooling @ 45%: Post warmup repooling', diff saved to https://phabricator.wikimedia.org/P53660 and previous config saved to /var/cache/conftool/dbconfig/20231121-090516-arnaudb.json |
[production] |