|
2022-06-15
ยง
|
| 13:27 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 13:27 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 13:26 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 13:25 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1034 (re)pooling @ 50%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29825 and previous config saved to /var/cache/conftool/dbconfig/20220615-132500-root.json |
[production] |
| 13:24 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1033 (re)pooling @ 50%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29824 and previous config saved to /var/cache/conftool/dbconfig/20220615-132454-root.json |
[production] |
| 13:24 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1032 (re)pooling @ 50%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29823 and previous config saved to /var/cache/conftool/dbconfig/20220615-132450-root.json |
[production] |
| 13:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1135', diff saved to https://phabricator.wikimedia.org/P29822 and previous config saved to /var/cache/conftool/dbconfig/20220615-131820-marostegui.json |
[production] |
| 13:09 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1034 (re)pooling @ 25%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29821 and previous config saved to /var/cache/conftool/dbconfig/20220615-130956-root.json |
[production] |
| 13:09 |
<ayounsi@deploy1002> |
Finished deploy [netbox/deploy@7bbf659]: deploying v3.1 (duration: 01m 03s) |
[production] |
| 13:09 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1033 (re)pooling @ 25%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29820 and previous config saved to /var/cache/conftool/dbconfig/20220615-130951-root.json |
[production] |
| 13:09 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1032 (re)pooling @ 25%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29819 and previous config saved to /var/cache/conftool/dbconfig/20220615-130946-root.json |
[production] |
| 13:08 |
<ayounsi@deploy1002> |
Started deploy [netbox/deploy@7bbf659]: deploying v3.1 |
[production] |
| 13:04 |
<ayounsi@deploy1002> |
Finished deploy [netbox/deploy@7bbf659]: deploying v3.1 (duration: 01m 43s) |
[production] |
| 13:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1135', diff saved to https://phabricator.wikimedia.org/P29818 and previous config saved to /var/cache/conftool/dbconfig/20220615-130315-marostegui.json |
[production] |
| 13:02 |
<ayounsi@deploy1002> |
Started deploy [netbox/deploy@7bbf659]: deploying v3.1 |
[production] |
| 13:00 |
<volans@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on netbox2002.codfw.wmnet with reason: Netbox upgrade to 3.2 |
[production] |
| 13:00 |
<volans@cumin1001> |
START - Cookbook sre.hosts.downtime for 4:00:00 on netbox2002.codfw.wmnet with reason: Netbox upgrade to 3.2 |
[production] |
| 13:00 |
<volans@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on netbox1002.eqiad.wmnet with reason: Netbox upgrade to 3.2 |
[production] |
| 13:00 |
<volans@cumin1001> |
START - Cookbook sre.hosts.downtime for 4:00:00 on netbox1002.eqiad.wmnet with reason: Netbox upgrade to 3.2 |
[production] |
| 12:56 |
<ayounsi@deploy1002> |
Finished deploy [netbox/deploy@7bbf659]: deploying v2.11.12 (duration: 00m 58s) |
[production] |
| 12:55 |
<ayounsi@deploy1002> |
Started deploy [netbox/deploy@7bbf659]: deploying v2.11.12 |
[production] |
| 12:55 |
<ayounsi@deploy1002> |
Finished deploy [netbox/deploy@7bbf659]: deploying v2.11.12 (duration: 00m 05s) |
[production] |
| 12:55 |
<ayounsi@deploy1002> |
Started deploy [netbox/deploy@7bbf659]: deploying v2.11.12 |
[production] |
| 12:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1034 (re)pooling @ 10%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29817 and previous config saved to /var/cache/conftool/dbconfig/20220615-125452-root.json |
[production] |
| 12:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1033 (re)pooling @ 10%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29816 and previous config saved to /var/cache/conftool/dbconfig/20220615-125447-root.json |
[production] |
| 12:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1032 (re)pooling @ 10%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29815 and previous config saved to /var/cache/conftool/dbconfig/20220615-125442-root.json |
[production] |
| 12:51 |
<jbond@deploy1002> |
Finished deploy [netbox/deploy@7bbf659]: log (duration: 03m 12s) |
[production] |
| 12:48 |
<jbond@deploy1002> |
Started deploy [netbox/deploy@7bbf659]: log |
[production] |
| 12:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1135 (T310011)', diff saved to https://phabricator.wikimedia.org/P29813 and previous config saved to /var/cache/conftool/dbconfig/20220615-124810-marostegui.json |
[production] |
| 12:42 |
<moritzm> |
failover ganeti master in eqsin to ganeti5001 |
[production] |
| 12:42 |
<volans@cumin1001> |
END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) for 6:00:00 on netbox:443 with reason: Netbox upgrade to 3.2 T296452 |
[production] |
| 12:42 |
<volans@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on netbox:443 with reason: Netbox upgrade to 3.2 T296452 |
[production] |
| 12:39 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1034 (re)pooling @ 5%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29812 and previous config saved to /var/cache/conftool/dbconfig/20220615-123949-root.json |
[production] |
| 12:39 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1033 (re)pooling @ 5%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29811 and previous config saved to /var/cache/conftool/dbconfig/20220615-123943-root.json |
[production] |
| 12:39 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1032 (re)pooling @ 5%: After upgrade', diff saved to https://phabricator.wikimedia.org/P29810 and previous config saved to /var/cache/conftool/dbconfig/20220615-123938-root.json |
[production] |
| 12:34 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti5002.eqsin.wmnet |
[production] |
| 12:26 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti5002.eqsin.wmnet |
[production] |
| 12:25 |
<kart_> |
Updated cxserver to 2022-06-15-074244-production (T309266, T310116, T309384, T306963) |
[production] |
| 12:23 |
<kartik@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/cxserver: apply |
[production] |
| 12:23 |
<kartik@deploy1002> |
helmfile [codfw] START helmfile.d/services/cxserver: apply |
[production] |
| 12:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool es1032 es1033 es1034 for kernel upgrade', diff saved to https://phabricator.wikimedia.org/P29808 and previous config saved to /var/cache/conftool/dbconfig/20220615-122123-root.json |
[production] |
| 12:19 |
<kartik@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/cxserver: apply |
[production] |
| 12:19 |
<kartik@deploy1002> |
helmfile [eqiad] START helmfile.d/services/cxserver: apply |
[production] |
| 12:16 |
<kartik@deploy1002> |
helmfile [staging] DONE helmfile.d/services/cxserver: apply |
[production] |
| 12:16 |
<kartik@deploy1002> |
helmfile [staging] START helmfile.d/services/cxserver: apply |
[production] |
| 12:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1135 (T310011)', diff saved to https://phabricator.wikimedia.org/P29807 and previous config saved to /var/cache/conftool/dbconfig/20220615-121620-marostegui.json |
[production] |
| 12:16 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1135.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:16 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1135.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:14 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:14 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |