|
2023-04-05
ยง
|
| 11:31 |
<slyngshede@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on testvm2004.codfw.wmnet with reason: host reimage |
[production] |
| 11:30 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1120 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P46058 and previous config saved to /var/cache/conftool/dbconfig/20230405-113052-root.json |
[production] |
| 11:30 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1122 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P46057 and previous config saved to /var/cache/conftool/dbconfig/20230405-113031-root.json |
[production] |
| 11:29 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host mw1414.eqiad.wmnet |
[production] |
| 11:28 |
<hnowlan@deploy2002> |
helmfile [eqiad] START helmfile.d/services/thumbor: apply |
[production] |
| 11:28 |
<hnowlan@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/thumbor: apply |
[production] |
| 11:24 |
<hnowlan@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/thumbor: apply |
[production] |
| 11:23 |
<ladsgroup@deploy2002> |
Finished scap: Backport for [[gerrit:905609|Revert "Revert "Revert "Revert "mwscript: Switch to use run.php"""" (T326800)]] (duration: 08m 45s) |
[production] |
| 11:23 |
<hnowlan@deploy2002> |
helmfile [codfw] START helmfile.d/services/thumbor: apply |
[production] |
| 11:22 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1100 (re)pooling @ 2%: Repooling', diff saved to https://phabricator.wikimedia.org/P46056 and previous config saved to /var/cache/conftool/dbconfig/20230405-112240-root.json |
[production] |
| 11:22 |
<slyngshede@cumin1001> |
START - Cookbook sre.ganeti.reimage for host testvm2004.codfw.wmnet with OS bullseye |
[production] |
| 11:17 |
<hnowlan@deploy2002> |
helmfile [eqiad] START helmfile.d/services/thumbor: apply |
[production] |
| 11:17 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1107 (re)pooling @ 3%: Repooling', diff saved to https://phabricator.wikimedia.org/P46055 and previous config saved to /var/cache/conftool/dbconfig/20230405-111742-root.json |
[production] |
| 11:17 |
<hnowlan@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/thumbor: apply |
[production] |
| 11:17 |
<hnowlan@deploy2002> |
helmfile [codfw] START helmfile.d/services/thumbor: apply |
[production] |
| 11:16 |
<ladsgroup@deploy2002> |
ladsgroup: Backport for [[gerrit:905609|Revert "Revert "Revert "Revert "mwscript: Switch to use run.php"""" (T326800)]] synced to the testservers: mwdebug1001.eqiad.wmnet, mwdebug2002.codfw.wmnet, mwdebug1002.eqiad.wmnet, mwdebug2001.codfw.wmnet |
[production] |
| 11:15 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1122 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P46054 and previous config saved to /var/cache/conftool/dbconfig/20230405-111527-root.json |
[production] |
| 11:15 |
<ladsgroup@deploy2002> |
Started scap: Backport for [[gerrit:905609|Revert "Revert "Revert "Revert "mwscript: Switch to use run.php"""" (T326800)]] |
[production] |
| 11:14 |
<hnowlan@deploy2002> |
helmfile [codfw] START helmfile.d/services/thumbor: apply |
[production] |
| 11:12 |
<hnowlan@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/thumbor: apply |
[production] |
| 11:12 |
<moritzm> |
installing systemd security updates on buster |
[production] |
| 11:12 |
<slyngshede@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reimage (exit_code=0) for host testvm2002.codfw.wmnet with OS bullseye |
[production] |
| 11:10 |
<hnowlan@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/thumbor: apply |
[production] |
| 11:07 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Set db1100 with 1% weight', diff saved to https://phabricator.wikimedia.org/P46053 and previous config saved to /var/cache/conftool/dbconfig/20230405-110717-root.json |
[production] |
| 11:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Promote db1130 to s5 primary T331302', diff saved to https://phabricator.wikimedia.org/P46052 and previous config saved to /var/cache/conftool/dbconfig/20230405-110530-root.json |
[production] |
| 11:05 |
<marostegui> |
Starting s5 eqiad failover from db1100 to db1130 - T331302 |
[production] |
| 11:02 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1107 (re)pooling @ 2%: Repooling', diff saved to https://phabricator.wikimedia.org/P46051 and previous config saved to /var/cache/conftool/dbconfig/20230405-110237-root.json |
[production] |
| 11:00 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1122 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P46050 and previous config saved to /var/cache/conftool/dbconfig/20230405-110022-root.json |
[production] |
| 11:00 |
<cgoubert@deploy2002> |
helmfile [staging] DONE helmfile.d/services/termbox: apply |
[production] |
| 11:00 |
<cgoubert@deploy2002> |
helmfile [staging] START helmfile.d/services/termbox: apply |
[production] |
| 10:59 |
<slyngshede@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on testvm2002.codfw.wmnet with reason: host reimage |
[production] |
| 10:59 |
<hnowlan@deploy2002> |
helmfile [eqiad] START helmfile.d/services/thumbor: apply |
[production] |
| 10:56 |
<hnowlan@deploy2002> |
helmfile [codfw] START helmfile.d/services/thumbor: apply |
[production] |
| 10:56 |
<slyngshede@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on testvm2002.codfw.wmnet with reason: host reimage |
[production] |
| 10:50 |
<hnowlan@deploy2002> |
helmfile [staging] DONE helmfile.d/services/thumbor: apply |
[production] |
| 10:50 |
<hnowlan@deploy2002> |
helmfile [staging] START helmfile.d/services/thumbor: apply |
[production] |
| 10:50 |
<hnowlan@deploy2002> |
helmfile [eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 10:49 |
<hnowlan@deploy2002> |
helmfile [eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 10:48 |
<hnowlan@deploy2002> |
helmfile [codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 10:48 |
<hnowlan@deploy2002> |
helmfile [codfw] START helmfile.d/admin 'apply'. |
[production] |
| 10:47 |
<slyngshede@cumin1001> |
START - Cookbook sre.ganeti.reimage for host testvm2002.codfw.wmnet with OS bullseye |
[production] |
| 10:47 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1107 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P46049 and previous config saved to /var/cache/conftool/dbconfig/20230405-104732-root.json |
[production] |
| 10:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1122 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P46048 and previous config saved to /var/cache/conftool/dbconfig/20230405-104517-root.json |
[production] |
| 10:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Set db1130 with weight 0 T331302', diff saved to https://phabricator.wikimedia.org/P46047 and previous config saved to /var/cache/conftool/dbconfig/20230405-104422-marostegui.json |
[production] |
| 10:44 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 24 hosts with reason: Primary switchover s5 T331302 |
[production] |
| 10:43 |
<hnowlan@deploy2002> |
helmfile [staging-codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 10:43 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 24 hosts with reason: Primary switchover s5 T331302 |
[production] |
| 10:43 |
<hnowlan@deploy2002> |
helmfile [staging-codfw] START helmfile.d/admin 'apply'. |
[production] |
| 10:41 |
<hnowlan@deploy2002> |
helmfile [staging-eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 10:40 |
<hnowlan@deploy2002> |
helmfile [staging-eqiad] START helmfile.d/admin 'apply'. |
[production] |