|
2024-06-10
ยง
|
| 14:11 |
<oblivian@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mw-debug: apply |
[production] |
| 14:10 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/mw-debug: apply |
[production] |
| 13:59 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T364069)', diff saved to https://phabricator.wikimedia.org/P64535 and previous config saved to /var/cache/conftool/dbconfig/20240610-135914-marostegui.json |
[production] |
| 13:57 |
<bking@cumin2002> |
END (PASS) - Cookbook sre.elasticsearch.ban (exit_code=0) Banning hosts: elastic1107.eqiad.wmnet for T348977 - bking@cumin2002 |
[production] |
| 13:57 |
<bking@cumin2002> |
START - Cookbook sre.elasticsearch.ban Banning hosts: elastic1107.eqiad.wmnet for T348977 - bking@cumin2002 |
[production] |
| 13:57 |
<bking@cumin2002> |
END (FAIL) - Cookbook sre.elasticsearch.ban (exit_code=99) Banning hosts: elastic1107 for T348977 - bking@cumin2002 |
[production] |
| 13:57 |
<bking@cumin2002> |
START - Cookbook sre.elasticsearch.ban Banning hosts: elastic1107 for T348977 - bking@cumin2002 |
[production] |
| 13:50 |
<fabfur@cumin1002> |
conftool action : set/pooled=yes; selector: name=cp4047.ulsfo.wmnet |
[production] |
| 13:49 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/superset: apply |
[production] |
| 13:48 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/superset: apply |
[production] |
| 13:47 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/superset-next: apply |
[production] |
| 13:47 |
<taavi@cumin1002> |
END (PASS) - Cookbook sre.loadbalancer.restart-pybal (exit_code=0) rolling-restart of pybal on A:lvs-secondary-eqiad or A:lvs-low-traffic-eqiad and A:lvs |
[production] |
| 13:47 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/superset-next: apply |
[production] |
| 13:46 |
<taavi@cumin1002> |
START - Cookbook sre.loadbalancer.restart-pybal rolling-restart of pybal on A:lvs-secondary-eqiad or A:lvs-low-traffic-eqiad and A:lvs |
[production] |
| 13:43 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/echoserver: apply |
[production] |
| 13:43 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/echoserver: apply |
[production] |
| 13:42 |
<elukey@deploy1002> |
helmfile [codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 13:42 |
<elukey@deploy1002> |
helmfile [codfw] START helmfile.d/admin 'apply'. |
[production] |
| 13:37 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/spark-history: apply |
[production] |
| 13:36 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/spark-history: apply |
[production] |
| 13:36 |
<elukey> |
move recommendation-api on wikikube to prometheus metrics (offboarded from statsd) - T205870 |
[production] |
| 13:36 |
<elukey@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/recommendation-api: sync |
[production] |
| 13:35 |
<elukey@deploy1002> |
helmfile [eqiad] START helmfile.d/services/recommendation-api: sync |
[production] |
| 13:34 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/spark-history: apply |
[production] |
| 13:34 |
<brouberol@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/spark-history: apply |
[production] |
| 13:30 |
<marostegui> |
dbmaint codfw s4 deploy schema change on db2140 T364069 |
[production] |
| 13:29 |
<taavi> |
taavi@mw1447 ~ $ sudo /usr/local/sbin/restart-php-fpm-all php7.4-fpm 9223372036854775807 # leftover from me restarting LVS during deployment |
[production] |
| 13:28 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2140.codfw.wmnet with reason: Long schema change |
[production] |
| 13:28 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2140.codfw.wmnet with reason: Long schema change |
[production] |
| 13:27 |
<elukey@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/recommendation-api: sync |
[production] |
| 13:26 |
<elukey@deploy1002> |
helmfile [codfw] START helmfile.d/services/recommendation-api: sync |
[production] |
| 13:26 |
<ladsgroup@cumin1002> |
dbctl commit (dc=all): 'Depooling db2203 (T352010)', diff saved to https://phabricator.wikimedia.org/P64534 and previous config saved to /var/cache/conftool/dbconfig/20240610-132619-ladsgroup.json |
[production] |
| 13:26 |
<ladsgroup@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2203.codfw.wmnet with reason: Maintenance |
[production] |
| 13:25 |
<ladsgroup@cumin1002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2203.codfw.wmnet with reason: Maintenance |
[production] |
| 13:25 |
<elukey@deploy1002> |
helmfile [staging] DONE helmfile.d/services/recommendation-api: sync |
[production] |
| 13:25 |
<elukey@deploy1002> |
helmfile [staging] START helmfile.d/services/recommendation-api: sync |
[production] |
| 13:20 |
<ladsgroup@deploy1002> |
Finished scap: Backport for [[gerrit:1041044|[huwiki] Add "suppressredirect" user right to editor user group (T366438)]] (duration: 15m 05s) |
[production] |
| 13:19 |
<fabfur@cumin1002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp4047.ulsfo.wmnet |
[production] |
| 13:18 |
<taavi@cumin1002> |
END (FAIL) - Cookbook sre.loadbalancer.restart-pybal (exit_code=99) rolling-restart of pybal on A:lvs-secondary-eqiad or A:lvs-low-traffic-eqiad and A:lvs |
[production] |
| 13:18 |
<taavi@cumin1002> |
START - Cookbook sre.loadbalancer.restart-pybal rolling-restart of pybal on A:lvs-secondary-eqiad or A:lvs-low-traffic-eqiad and A:lvs |
[production] |
| 13:13 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti2021.codfw.wmnet |
[production] |
| 13:13 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti2021.codfw.wmnet |
[production] |
| 13:13 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1018.eqiad.wmnet |
[production] |
| 13:13 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1018.eqiad.wmnet |
[production] |
| 13:11 |
<taavi> |
restarting eqiad low-traffic LVS for https://gerrit.wikimedia.org/r/c/operations/puppet/+/941459 |
[production] |
| 13:11 |
<ladsgroup@deploy1002> |
ladsgroup and gergesshamon: Continuing with sync |
[production] |
| 13:10 |
<fabfur@cumin1002> |
START - Cookbook sre.hosts.reboot-single for host cp4047.ulsfo.wmnet |
[production] |
| 13:10 |
<elukey@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 13:09 |
<fabfur@cumin1002> |
conftool action : set/pooled=no; selector: name=cp4047.ulsfo.wmnet |
[production] |
| 13:09 |
<fabfur> |
rebooting cp4047 (T366555) |
[production] |