2023-09-27
ยง
|
11:23 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'Depooling db2182 (T343198)', diff saved to https://phabricator.wikimedia.org/P52687 and previous config saved to /var/cache/conftool/dbconfig/20230927-112342-arnaudb.json |
[production] |
11:23 |
<arnaudb@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2182.codfw.wmnet with reason: Maintenance |
[production] |
11:23 |
<arnaudb@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2182.codfw.wmnet with reason: Maintenance |
[production] |
11:23 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317 (T343198)', diff saved to https://phabricator.wikimedia.org/P52686 and previous config saved to /var/cache/conftool/dbconfig/20230927-112320-arnaudb.json |
[production] |
11:08 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317', diff saved to https://phabricator.wikimedia.org/P52685 and previous config saved to /var/cache/conftool/dbconfig/20230927-110813-arnaudb.json |
[production] |
10:55 |
<fnegri@cloudcumin1001> |
END (PASS) - Cookbook wmcs.vps.create_instance_with_prefix (exit_code=0) with prefix 'bastion-restricted-eqiad1' (T347428) |
[bastion] |
10:53 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317', diff saved to https://phabricator.wikimedia.org/P52684 and previous config saved to /var/cache/conftool/dbconfig/20230927-105306-arnaudb.json |
[production] |
10:52 |
<fnegri@cloudcumin1001> |
START - Cookbook wmcs.vps.create_instance_with_prefix with prefix 'bastion-restricted-eqiad1' (T347428) |
[bastion] |
10:46 |
<cgoubert@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/mw-web: apply |
[production] |
10:46 |
<cgoubert@deploy2002> |
helmfile [codfw] START helmfile.d/services/mw-web: apply |
[production] |
10:45 |
<cgoubert@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: apply |
[production] |
10:45 |
<cgoubert@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-web: apply |
[production] |
10:40 |
<isaranto@deploy2002> |
helmfile [ml-serve-eqiad] 'sync' command on namespace 'ores-legacy' for release 'main' . |
[production] |
10:39 |
<isaranto@deploy2002> |
helmfile [ml-serve-codfw] 'sync' command on namespace 'ores-legacy' for release 'main' . |
[production] |
10:39 |
<isaranto@deploy2002> |
helmfile [ml-staging-codfw] 'sync' command on namespace 'ores-legacy' for release 'main' . |
[production] |
10:38 |
<arnaudb@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317 (T343198)', diff saved to https://phabricator.wikimedia.org/P52683 and previous config saved to /var/cache/conftool/dbconfig/20230927-103800-arnaudb.json |
[production] |
10:27 |
<cgoubert@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: apply |
[production] |
10:27 |
<cgoubert@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-web: apply |
[production] |
10:27 |
<cgoubert@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/mw-web: apply |
[production] |
10:27 |
<cgoubert@deploy2002> |
helmfile [codfw] START helmfile.d/services/mw-web: apply |
[production] |
09:48 |
<jayme@cumin1001> |
conftool action : set/pooled=no; selector: name=kubernetes1013.* |
[production] |
09:43 |
<claime> |
Bumping mw-on-k8s traffic to 8% - T346422 |
[production] |
09:36 |
<taavi> |
move maintain-dbusers to cloudcontrol1005 |
[admin] |
09:36 |
<jayme> |
cordoning kubernetes1013 for debug porposes |
[production] |
09:33 |
<taavi> |
update CR firewall policy, gerrit 961336 |
[production] |
09:15 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on db2109.codfw.wmnet with reason: Host crashed |
[production] |
09:14 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 7 days, 0:00:00 on db2109.codfw.wmnet with reason: Host crashed |
[production] |
09:10 |
<gmodena@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
09:10 |
<gmodena@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
09:08 |
<gmodena@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
09:08 |
<gmodena@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
09:05 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3:00:00 on 15 hosts with reason: Kafka mirror issues on jumbo |
[production] |
09:05 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.downtime for 3:00:00 on 15 hosts with reason: Kafka mirror issues on jumbo |
[production] |
08:55 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on db2109.codfw.wmnet with reason: Host crashed |
[production] |
08:54 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 7 days, 0:00:00 on db2109.codfw.wmnet with reason: Host crashed |
[production] |
08:44 |
<jiji@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/mathoid: apply |
[production] |
08:44 |
<jiji@deploy2002> |
helmfile [codfw] START helmfile.d/services/mathoid: apply |
[production] |
08:44 |
<jiji@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mathoid: apply |
[production] |
08:44 |
<jiji@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mathoid: apply |
[production] |
08:29 |
<elukey> |
`elukey@cumin1001:~$ sudo cumin 'kafka-jumbo10[01-05]*' 'systemctl start kafka-mirror' -b 1 -s 30` |
[analytics] |
08:28 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 15 hosts with reason: Kafka mirror issues on jumbo |
[production] |
08:28 |
<elukey> |
`elukey@cumin1001:~$ sudo cumin 'kafka-jumbo10[06-15]*' 'systemctl stop kafka-mirror'` |
[analytics] |
08:28 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 15 hosts with reason: Kafka mirror issues on jumbo |
[production] |
08:21 |
<vgutierrez> |
update HAProxy to version 2.7.10 in cp4051 - T317799 |
[production] |
08:13 |
<elukey> |
slowly start mirror maker on one instance at the time on all jumbo nodes |
[analytics] |
08:11 |
<elukey> |
start kafka mirror on jumbo1002 |
[analytics] |
08:10 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:30:00 on 15 hosts with reason: Kafka mirror issues on jumbo |
[production] |
08:10 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.downtime for 0:30:00 on 15 hosts with reason: Kafka mirror issues on jumbo |
[production] |
08:08 |
<elukey> |
stop all mirror maker on jumbo, start only one on jumbo1001 |
[analytics] |
07:47 |
<elukey> |
roll restart mirror maker instances on kafka jumbo |
[analytics] |