|
2021-01-20
ยง
|
| 15:46 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'test' . |
[production] |
| 15:46 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'canary' . |
[production] |
| 15:43 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'canary' . |
[production] |
| 15:43 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'main' . |
[production] |
| 15:43 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'test' . |
[production] |
| 15:40 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2056.codfw.wmnet |
[production] |
| 15:34 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'test' . |
[production] |
| 15:34 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'canary' . |
[production] |
| 15:34 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'main' . |
[production] |
| 15:32 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db1109 (re)pooling @ 75%: Reboot T272255', diff saved to https://phabricator.wikimedia.org/P13857 and previous config saved to /var/cache/conftool/dbconfig/20210120-153223-kormat.json |
[production] |
| 15:32 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'canary' . |
[production] |
| 15:32 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'main' . |
[production] |
| 15:32 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'test' . |
[production] |
| 15:24 |
<brennen@deploy1001> |
rebuilt and synchronized wikiversions files: group0 wikis to 1.36.0-wmf.27 |
[production] |
| 15:23 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2054.codfw.wmnet |
[production] |
| 15:18 |
<brennen> |
1.36.0-wmf.27 train unblocked, proceeding to group0 (T271341) |
[production] |
| 15:17 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2054.codfw.wmnet |
[production] |
| 15:17 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db1109 (re)pooling @ 50%: Reboot T272255', diff saved to https://phabricator.wikimedia.org/P13856 and previous config saved to /var/cache/conftool/dbconfig/20210120-151719-kormat.json |
[production] |
| 15:17 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2053.codfw.wmnet |
[production] |
| 15:15 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db1076 (re)pooling @ 100%: Reboot T272255', diff saved to https://phabricator.wikimedia.org/P13855 and previous config saved to /var/cache/conftool/dbconfig/20210120-151555-kormat.json |
[production] |
| 15:12 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2053.codfw.wmnet |
[production] |
| 15:08 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2052.codfw.wmnet |
[production] |
| 15:02 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2052.codfw.wmnet |
[production] |
| 15:02 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db1109 (re)pooling @ 25%: Reboot T272255', diff saved to https://phabricator.wikimedia.org/P13854 and previous config saved to /var/cache/conftool/dbconfig/20210120-150216-kormat.json |
[production] |
| 15:01 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2051.codfw.wmnet |
[production] |
| 15:00 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db1076 (re)pooling @ 66%: Reboot T272255', diff saved to https://phabricator.wikimedia.org/P13853 and previous config saved to /var/cache/conftool/dbconfig/20210120-150051-kormat.json |
[production] |
| 14:59 |
<elukey@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'kube-system' for release 'rbac-deploy-clusterrole' . |
[production] |
| 14:57 |
<otto@deploy1001> |
Synchronized wmf-config/InitialiseSettings.php: Migrate QuickSurveys schemas to EventGate on all wikis - T271165, T271166 (duration: 01m 05s) |
[production] |
| 14:56 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db1109 depooling: Rebooting for T272255', diff saved to https://phabricator.wikimedia.org/P13852 and previous config saved to /var/cache/conftool/dbconfig/20210120-145605-kormat.json |
[production] |
| 14:56 |
<kormat@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:30:00 on db1109.eqiad.wmnet with reason: Rebooting for T272255 |
[production] |
| 14:56 |
<kormat@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:30:00 on db1109.eqiad.wmnet with reason: Rebooting for T272255 |
[production] |
| 14:55 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2051.codfw.wmnet |
[production] |
| 14:53 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2050.codfw.wmnet |
[production] |
| 14:47 |
<otto@deploy1001> |
Synchronized wmf-config/InitialiseSettings.php: Migrate QuickSurveys schemas to EventGate on testwiki - T271165, T271166 (duration: 01m 06s) |
[production] |
| 14:46 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2050.codfw.wmnet |
[production] |
| 14:46 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2049.codfw.wmnet |
[production] |
| 14:45 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db1076 (re)pooling @ 33%: Reboot T272255', diff saved to https://phabricator.wikimedia.org/P13851 and previous config saved to /var/cache/conftool/dbconfig/20210120-144547-kormat.json |
[production] |
| 14:40 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2049.codfw.wmnet |
[production] |
| 14:40 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2048.codfw.wmnet |
[production] |
| 14:34 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2048.codfw.wmnet |
[production] |
| 14:32 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2047.codfw.wmnet |
[production] |
| 14:32 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'test' . |
[production] |
| 14:32 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'canary' . |
[production] |
| 14:32 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'similar-users' for release 'main' . |
[production] |
| 14:26 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single for host ms-be2047.codfw.wmnet |
[production] |
| 14:26 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db1076 depooling: Rebooting for T272255', diff saved to https://phabricator.wikimedia.org/P13850 and previous config saved to /var/cache/conftool/dbconfig/20210120-142636-kormat.json |
[production] |
| 14:26 |
<kormat@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:30:00 on db1076.eqiad.wmnet with reason: Rebooting for T272255 |
[production] |
| 14:26 |
<kormat@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:30:00 on db1076.eqiad.wmnet with reason: Rebooting for T272255 |
[production] |
| 14:26 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ms-be2046.codfw.wmnet |
[production] |
| 14:21 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'Repooling after reboot. T272255', diff saved to https://phabricator.wikimedia.org/P13849 and previous config saved to /var/cache/conftool/dbconfig/20210120-142139-kormat.json |
[production] |