|
2024-02-07
ยง
|
| 13:52 |
<klausman@deploy2002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'apply'. |
[production] |
| 13:48 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'es2024 (re)pooling @ 15%: kernel upgrade done', diff saved to https://phabricator.wikimedia.org/P56436 and previous config saved to /var/cache/conftool/dbconfig/20240207-134801-arnaudb.json |
[production] |
| 13:39 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'db2105 (re)pooling @ 30%: kernel upgrade done', diff saved to https://phabricator.wikimedia.org/P56435 and previous config saved to /var/cache/conftool/dbconfig/20240207-133907-arnaudb.json |
[production] |
| 13:32 |
<jmm@cumin2002> |
END (FAIL) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=1) rolling reboot on A:ncredir |
[production] |
| 13:26 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'T344589 - depool es2024', diff saved to https://phabricator.wikimedia.org/P56434 and previous config saved to /var/cache/conftool/dbconfig/20240207-132559-arnaudb.json |
[production] |
| 13:25 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on es2024.codfw.wmnet with reason: T344589 - kernel upgrade |
[production] |
| 13:25 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on es2024.codfw.wmnet with reason: T344589 - kernel upgrade |
[production] |
| 13:24 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'db2105 (re)pooling @ 15%: kernel upgrade done', diff saved to https://phabricator.wikimedia.org/P56433 and previous config saved to /var/cache/conftool/dbconfig/20240207-132402-arnaudb.json |
[production] |
| 12:46 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'T344589 - depool db2105', diff saved to https://phabricator.wikimedia.org/P56432 and previous config saved to /var/cache/conftool/dbconfig/20240207-124605-arnaudb.json |
[production] |
| 12:45 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on db2105.codfw.wmnet with reason: T344589 - kernel upgrade |
[production] |
| 12:45 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on db2105.codfw.wmnet with reason: T344589 - kernel upgrade |
[production] |
| 12:44 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2190 (T355609)', diff saved to https://phabricator.wikimedia.org/P56431 and previous config saved to /var/cache/conftool/dbconfig/20240207-124409-marostegui.json |
[production] |
| 12:35 |
<hnowlan@cumin2002> |
END (PASS) - Cookbook sre.loadbalancer.restart-pybal (exit_code=0) rolling-restart of pybal on P{lvs1019*,lvs2013*} and A:lvs (T334488) |
[production] |
| 12:34 |
<hnowlan@cumin2002> |
START - Cookbook sre.loadbalancer.restart-pybal rolling-restart of pybal on P{lvs1019*,lvs2013*} and A:lvs (T334488) |
[production] |
| 12:33 |
<jmm@cumin2002> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on A:ncredir |
[production] |
| 12:32 |
<hnowlan@cumin2002> |
END (PASS) - Cookbook sre.loadbalancer.restart-pybal (exit_code=0) rolling-restart of pybal on P{lvs1020*,lvs2014*} and A:lvs (T334488) |
[production] |
| 12:31 |
<hnowlan@cumin2002> |
START - Cookbook sre.loadbalancer.restart-pybal rolling-restart of pybal on P{lvs1020*,lvs2014*} and A:lvs (T334488) |
[production] |
| 12:31 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host lvs1016.eqiad.wmnet |
[production] |
| 12:29 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2190', diff saved to https://phabricator.wikimedia.org/P56430 and previous config saved to /var/cache/conftool/dbconfig/20240207-122903-marostegui.json |
[production] |
| 12:25 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host lvs1016.eqiad.wmnet |
[production] |
| 12:25 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host lvs1015.eqiad.wmnet |
[production] |
| 12:19 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host lvs1015.eqiad.wmnet |
[production] |
| 12:18 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host lvs1014.eqiad.wmnet |
[production] |
| 12:17 |
<claime> |
trafficserver: move 40% of traffic to mw on k8s - T355532 |
[production] |
| 12:14 |
<cgoubert@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/mw-web: apply |
[production] |
| 12:14 |
<cgoubert@deploy2002> |
helmfile [codfw] START helmfile.d/services/mw-web: apply |
[production] |
| 12:14 |
<cgoubert@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: apply |
[production] |
| 12:14 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host lvs1014.eqiad.wmnet |
[production] |
| 12:14 |
<cgoubert@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-web: apply |
[production] |
| 12:13 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2190', diff saved to https://phabricator.wikimedia.org/P56429 and previous config saved to /var/cache/conftool/dbconfig/20240207-121356-marostegui.json |
[production] |
| 12:13 |
<cgoubert@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/mw-api-ext: apply |
[production] |
| 12:12 |
<cgoubert@deploy2002> |
helmfile [codfw] START helmfile.d/services/mw-api-ext: apply |
[production] |
| 12:12 |
<cgoubert@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-ext: apply |
[production] |
| 12:12 |
<cgoubert@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-api-ext: apply |
[production] |
| 12:12 |
<claime> |
mw-web, mw-api-ext: Raise replicas for 40% traffic - T355532 |
[production] |
| 12:02 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host lvs1013.eqiad.wmnet |
[production] |
| 12:02 |
<volans> |
uploaded debmonitor-client_0.3.5 to apt.wikimedia.org buster-wikimedia,bullseye-wikimedia,bookworm-wikimedia |
[production] |
| 12:01 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.restart-reboot-config-master (exit_code=0) rolling reboot on A:config-master-eqiad |
[production] |
| 11:58 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2190 (T355609)', diff saved to https://phabricator.wikimedia.org/P56428 and previous config saved to /var/cache/conftool/dbconfig/20240207-115849-marostegui.json |
[production] |
| 11:56 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host lvs1013.eqiad.wmnet |
[production] |
| 11:56 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) config-master.discovery.wmnet. on all recursors |
[production] |
| 11:56 |
<jmm@cumin2002> |
START - Cookbook sre.dns.wipe-cache config-master.discovery.wmnet. on all recursors |
[production] |
| 11:56 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.restart-reboot-config-master rolling reboot on A:config-master-eqiad |
[production] |
| 11:53 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.restart-reboot-config-master (exit_code=0) rolling reboot on A:config-master-codfw |
[production] |
| 11:49 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) config-master.discovery.wmnet. on all recursors |
[production] |
| 11:49 |
<jmm@cumin2002> |
START - Cookbook sre.dns.wipe-cache config-master.discovery.wmnet. on all recursors |
[production] |
| 11:48 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.restart-reboot-config-master rolling reboot on A:config-master-codfw |
[production] |
| 11:33 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2190 (T355609)', diff saved to https://phabricator.wikimedia.org/P56427 and previous config saved to /var/cache/conftool/dbconfig/20240207-113339-marostegui.json |
[production] |
| 11:33 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2190.codfw.wmnet with reason: Maintenance |
[production] |
| 11:33 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2190.codfw.wmnet with reason: Maintenance |
[production] |