|
2024-05-06
ยง
|
| 17:05 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2212', diff saved to https://phabricator.wikimedia.org/P61973 and previous config saved to /var/cache/conftool/dbconfig/20240506-170556-marostegui.json |
[production] |
| 16:52 |
<sukhe> |
sudo cumin 'A:ncredir' 'run-puppet-agent --enable "merging CR 1028514"' |
[production] |
| 16:50 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2212', diff saved to https://phabricator.wikimedia.org/P61972 and previous config saved to /var/cache/conftool/dbconfig/20240506-165048-marostegui.json |
[production] |
| 16:45 |
<sukhe> |
disable puppet on A:ncredir to merge CR 1028514 |
[production] |
| 16:38 |
<jayme@cumin1002> |
END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for mw2382.codfw.wmnet |
[production] |
| 16:38 |
<jayme@cumin1002> |
START - Cookbook sre.hosts.remove-downtime for mw2382.codfw.wmnet |
[production] |
| 16:37 |
<jayme@cumin1002> |
conftool action : set/pooled=yes; selector: name=mw2382.codfw.wmnet |
[production] |
| 16:35 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2212 (T361627)', diff saved to https://phabricator.wikimedia.org/P61971 and previous config saved to /var/cache/conftool/dbconfig/20240506-163540-marostegui.json |
[production] |
| 16:25 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2212 (T361627)', diff saved to https://phabricator.wikimedia.org/P61970 and previous config saved to /var/cache/conftool/dbconfig/20240506-162528-marostegui.json |
[production] |
| 16:25 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2212.codfw.wmnet with reason: Maintenance |
[production] |
| 16:25 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on db2212.codfw.wmnet with reason: Maintenance |
[production] |
| 16:16 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2202.codfw.wmnet with reason: Maintenance |
[production] |
| 16:16 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on db2202.codfw.wmnet with reason: Maintenance |
[production] |
| 16:16 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2188 (T361627)', diff saved to https://phabricator.wikimedia.org/P61969 and previous config saved to /var/cache/conftool/dbconfig/20240506-161624-marostegui.json |
[production] |
| 16:01 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2188', diff saved to https://phabricator.wikimedia.org/P61968 and previous config saved to /var/cache/conftool/dbconfig/20240506-160116-marostegui.json |
[production] |
| 15:54 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2021 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P61967 and previous config saved to /var/cache/conftool/dbconfig/20240506-155420-root.json |
[production] |
| 15:46 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2188', diff saved to https://phabricator.wikimedia.org/P61966 and previous config saved to /var/cache/conftool/dbconfig/20240506-154608-marostegui.json |
[production] |
| 15:39 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2021 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P61965 and previous config saved to /var/cache/conftool/dbconfig/20240506-153914-root.json |
[production] |
| 15:31 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2188 (T361627)', diff saved to https://phabricator.wikimedia.org/P61964 and previous config saved to /var/cache/conftool/dbconfig/20240506-153101-marostegui.json |
[production] |
| 15:25 |
<swfrench@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mathoid: apply |
[production] |
| 15:24 |
<swfrench@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mathoid: apply |
[production] |
| 15:24 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2021 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P61963 and previous config saved to /var/cache/conftool/dbconfig/20240506-152408-root.json |
[production] |
| 15:20 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2188 (T361627)', diff saved to https://phabricator.wikimedia.org/P61962 and previous config saved to /var/cache/conftool/dbconfig/20240506-152040-marostegui.json |
[production] |
| 15:20 |
<urbanecm@deploy1002> |
Finished scap: Backport for [[gerrit:1027744|userOptions.php: Actually batch deletion (T364311)]] (duration: 16m 51s) |
[production] |
| 15:20 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2188.codfw.wmnet with reason: Maintenance |
[production] |
| 15:20 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on db2188.codfw.wmnet with reason: Maintenance |
[production] |
| 15:20 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2176 (T361627)', diff saved to https://phabricator.wikimedia.org/P61961 and previous config saved to /var/cache/conftool/dbconfig/20240506-152016-marostegui.json |
[production] |
| 15:17 |
<urbanecm> |
[urbanecm@mwmaint1002 ~]$ foreachwikiindblist s{3-8} userOptions.php --delete rcenhancedfilters-seen-tour # T364269 |
[production] |
| 15:16 |
<urbanecm> |
[urbanecm@mwmaint1002 ~]$ foreachwikiindblist s2 userOptions.php --delete rcenhancedfilters-seen-tour # T364269 |
[production] |
| 15:16 |
<urbanecm> |
[urbanecm@mwmaint1002 ~]$ mwscript userOptions.php --wiki=enwiki --delete rcenhancedfilters-seen-tour # T364269 |
[production] |
| 15:12 |
<swfrench@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mathoid: apply |
[production] |
| 15:11 |
<brouberol@cumin2002> |
END (PASS) - Cookbook sre.elasticsearch.rolling-operation (exit_code=0) Operation.RESTART (3 nodes at a time) for ElasticSearch cluster cloudelastic: restart to pick up new JDK - brouberol@cumin2002 - T363975 |
[production] |
| 15:09 |
<swfrench@deploy1002> |
helmfile [codfw] START helmfile.d/services/mathoid: apply |
[production] |
| 15:09 |
<urbanecm> |
mwmaint1002: mwscript userOptions.php --wiki=loginwiki --delete rcenhancedfilters-seen-tour # T364269 |
[production] |
| 15:09 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2021 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P61960 and previous config saved to /var/cache/conftool/dbconfig/20240506-150902-root.json |
[production] |
| 15:05 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2176', diff saved to https://phabricator.wikimedia.org/P61959 and previous config saved to /var/cache/conftool/dbconfig/20240506-150508-marostegui.json |
[production] |
| 15:03 |
<urbanecm@deploy1002> |
Started scap: Backport for [[gerrit:1027744|userOptions.php: Actually batch deletion (T364311)]] |
[production] |
| 15:03 |
<swfrench@deploy1002> |
helmfile [staging] DONE helmfile.d/services/mathoid: apply |
[production] |
| 15:01 |
<swfrench@deploy1002> |
helmfile [staging] START helmfile.d/services/mathoid: apply |
[production] |
| 14:55 |
<moritzm> |
installing less security updates |
[production] |
| 14:54 |
<filippo@cumin1002> |
END (FAIL) - Cookbook sre.ganeti.makevm (exit_code=97) for new host prometheus7001.magru.wmnet |
[production] |
| 14:54 |
<filippo@cumin1002> |
END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host prometheus7001.magru.wmnet with OS bullseye |
[production] |
| 14:53 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2021 (re)pooling @ 10%: Repooling', diff saved to https://phabricator.wikimedia.org/P61958 and previous config saved to /var/cache/conftool/dbconfig/20240506-145356-root.json |
[production] |
| 14:51 |
<brouberol@cumin2002> |
START - Cookbook sre.elasticsearch.rolling-operation Operation.RESTART (3 nodes at a time) for ElasticSearch cluster cloudelastic: restart to pick up new JDK - brouberol@cumin2002 - T363975 |
[production] |
| 14:50 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2176', diff saved to https://phabricator.wikimedia.org/P61957 and previous config saved to /var/cache/conftool/dbconfig/20240506-145001-marostegui.json |
[production] |
| 14:46 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.puppet.migrate-host (exit_code=0) for host db2209.codfw.wmnet |
[production] |
| 14:45 |
<dcausse@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/cirrus-streaming-updater: apply |
[production] |
| 14:45 |
<urbanecm@deploy1002> |
Finished scap: Backport for [[gerrit:1027476|Revert "Release DT visual enhancements to all except Wikipedia/Commons/Wikidata"]] (duration: 18m 11s) |
[production] |
| 14:45 |
<dcausse@deploy1002> |
helmfile [codfw] START helmfile.d/services/cirrus-streaming-updater: apply |
[production] |
| 14:38 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2021 (re)pooling @ 5%: Repooling', diff saved to https://phabricator.wikimedia.org/P61956 and previous config saved to /var/cache/conftool/dbconfig/20240506-143850-root.json |
[production] |