|
2024-05-08
ยง
|
| 08:31 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on es2023.codfw.wmnet with reason: host reimage |
[production] |
| 08:31 |
<klausman@deploy1002> |
helmfile [ml-serve-eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 08:29 |
<ladsgroup@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1150.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:29 |
<ladsgroup@cumin1002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1150.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:29 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db1178 (re)pooling @ 10%: Repooling', diff saved to https://phabricator.wikimedia.org/P62032 and previous config saved to /var/cache/conftool/dbconfig/20240508-082917-root.json |
[production] |
| 08:24 |
<klausman@deploy1002> |
helmfile [ml-serve-codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 08:23 |
<klausman@deploy1002> |
helmfile [ml-serve-codfw] START helmfile.d/admin 'apply'. |
[production] |
| 08:22 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 08:22 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1177 (T361627)', diff saved to https://phabricator.wikimedia.org/P62031 and previous config saved to /var/cache/conftool/dbconfig/20240508-082231-marostegui.json |
[production] |
| 08:22 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2022 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P62030 and previous config saved to /var/cache/conftool/dbconfig/20240508-082202-root.json |
[production] |
| 08:21 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'apply'. |
[production] |
| 08:21 |
<jmm@cumin2002> |
START - Cookbook sre.puppet.migrate-host for host db2182.codfw.wmnet |
[production] |
| 08:20 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.puppet.migrate-host (exit_code=0) for host db2168.codfw.wmnet |
[production] |
| 08:16 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2025 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P62029 and previous config saved to /var/cache/conftool/dbconfig/20240508-081633-root.json |
[production] |
| 08:14 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db1178 (re)pooling @ 5%: Repooling', diff saved to https://phabricator.wikimedia.org/P62028 and previous config saved to /var/cache/conftool/dbconfig/20240508-081412-root.json |
[production] |
| 08:12 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.reimage for host es2023.codfw.wmnet with OS bookworm |
[production] |
| 08:08 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Give some weight to es5 master', diff saved to https://phabricator.wikimedia.org/P62027 and previous config saved to /var/cache/conftool/dbconfig/20240508-080848-marostegui.json |
[production] |
| 08:08 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depool es2023 T364443', diff saved to https://phabricator.wikimedia.org/P62026 and previous config saved to /var/cache/conftool/dbconfig/20240508-080812-root.json |
[production] |
| 08:06 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2022 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P62025 and previous config saved to /var/cache/conftool/dbconfig/20240508-080656-root.json |
[production] |
| 08:06 |
<marostegui> |
Starting es5 codfw failover from es2023 to es2024 - T364443 |
[production] |
| 08:03 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Set es2024 with weight 0 T364443', diff saved to https://phabricator.wikimedia.org/P62024 and previous config saved to /var/cache/conftool/dbconfig/20240508-080312-root.json |
[production] |
| 08:03 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 6 hosts with reason: Primary switchover es5 T364443 |
[production] |
| 08:02 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 6 hosts with reason: Primary switchover es5 T364443 |
[production] |
| 08:01 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2025 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P62023 and previous config saved to /var/cache/conftool/dbconfig/20240508-080128-root.json |
[production] |
| 07:59 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db1178 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P62022 and previous config saved to /var/cache/conftool/dbconfig/20240508-075906-root.json |
[production] |
| 07:57 |
<jmm@cumin2002> |
START - Cookbook sre.puppet.migrate-host for host db2168.codfw.wmnet |
[production] |
| 07:57 |
<Emperor> |
depool/restart/repool ms-fe1012 |
[production] |
| 07:56 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1177 (T361627)', diff saved to https://phabricator.wikimedia.org/P62021 and previous config saved to /var/cache/conftool/dbconfig/20240508-075635-marostegui.json |
[production] |
| 07:56 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db1177.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:56 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on db1177.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:56 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1172 (T361627)', diff saved to https://phabricator.wikimedia.org/P62020 and previous config saved to /var/cache/conftool/dbconfig/20240508-075610-marostegui.json |
[production] |
| 07:55 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.puppet.migrate-host (exit_code=0) for host db2150.codfw.wmnet |
[production] |
| 07:51 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2022 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P62019 and previous config saved to /var/cache/conftool/dbconfig/20240508-075150-root.json |
[production] |
| 07:46 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2025 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P62018 and previous config saved to /var/cache/conftool/dbconfig/20240508-074620-root.json |
[production] |
| 07:41 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1172', diff saved to https://phabricator.wikimedia.org/P62017 and previous config saved to /var/cache/conftool/dbconfig/20240508-074102-marostegui.json |
[production] |
| 07:36 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2022 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P62016 and previous config saved to /var/cache/conftool/dbconfig/20240508-073644-root.json |
[production] |
| 07:33 |
<jmm@cumin2002> |
START - Cookbook sre.puppet.migrate-host for host db2150.codfw.wmnet |
[production] |
| 07:31 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2025 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P62015 and previous config saved to /var/cache/conftool/dbconfig/20240508-073109-root.json |
[production] |
| 07:25 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1172', diff saved to https://phabricator.wikimedia.org/P62014 and previous config saved to /var/cache/conftool/dbconfig/20240508-072554-marostegui.json |
[production] |
| 07:21 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2022 (re)pooling @ 10%: Repooling', diff saved to https://phabricator.wikimedia.org/P62012 and previous config saved to /var/cache/conftool/dbconfig/20240508-072138-root.json |
[production] |
| 07:16 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2025 (re)pooling @ 10%: Repooling', diff saved to https://phabricator.wikimedia.org/P62011 and previous config saved to /var/cache/conftool/dbconfig/20240508-071604-root.json |
[production] |
| 07:10 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1172 (T361627)', diff saved to https://phabricator.wikimedia.org/P62010 and previous config saved to /var/cache/conftool/dbconfig/20240508-071047-marostegui.json |
[production] |
| 07:06 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2022 (re)pooling @ 5%: Repooling', diff saved to https://phabricator.wikimedia.org/P62009 and previous config saved to /var/cache/conftool/dbconfig/20240508-070632-root.json |
[production] |
| 07:01 |
<moritzm> |
uninstalling git-fat on buster hosts T364373 |
[production] |
| 07:00 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2025 (re)pooling @ 5%: Repooling', diff saved to https://phabricator.wikimedia.org/P62008 and previous config saved to /var/cache/conftool/dbconfig/20240508-070058-root.json |
[production] |
| 06:51 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2022 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P62007 and previous config saved to /var/cache/conftool/dbconfig/20240508-065127-root.json |
[production] |
| 06:45 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2025 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P62006 and previous config saved to /var/cache/conftool/dbconfig/20240508-064552-root.json |
[production] |
| 06:45 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1172 (T361627)', diff saved to https://phabricator.wikimedia.org/P62005 and previous config saved to /var/cache/conftool/dbconfig/20240508-064523-marostegui.json |
[production] |
| 06:45 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db1172.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:44 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on db1172.eqiad.wmnet with reason: Maintenance |
[production] |