|
2021-03-23
ยง
|
| 10:20 |
<jayme@deploy1002> |
helmfile [eqiad] Ran 'sync' command on namespace 'proton' for release 'production' . |
[production] |
| 10:20 |
<jayme@deploy1002> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventstreams' for release 'production' . |
[production] |
| 10:20 |
<jayme@deploy1002> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventstreams' for release 'canary' . |
[production] |
| 10:20 |
<jayme@deploy1002> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-main' for release 'canary' . |
[production] |
| 10:19 |
<jayme@deploy1002> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-main' for release 'production' . |
[production] |
| 10:19 |
<jayme@deploy1002> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'canary' . |
[production] |
| 10:19 |
<jayme@deploy1002> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'production' . |
[production] |
| 10:19 |
<jayme@deploy1002> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-analytics-external' for release 'production' . |
[production] |
| 10:19 |
<jayme@deploy1002> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-analytics-external' for release 'canary' . |
[production] |
| 10:19 |
<hashar@deploy1002> |
Pruned MediaWiki: 1.36.0-wmf.33 (duration: 01m 48s) |
[production] |
| 10:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1147 (re)pooling @ 25%: Slowly repool db1147 after schema change', diff saved to https://phabricator.wikimedia.org/P15030 and previous config saved to /var/cache/conftool/dbconfig/20210323-101836-root.json |
[production] |
| 10:16 |
<hashar@deploy1002> |
Pruned MediaWiki: 1.36.0-wmf.32 (duration: 14m 47s) |
[production] |
| 10:10 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes1005.eqiad.wmnet |
[production] |
| 10:02 |
<hashar> |
scap clean --delete 1.36.0-wmf.32 # T274940 |
[production] |
| 10:01 |
<hashar> |
Applied security patches for 1.36.0-wmf.36 # T274940 |
[production] |
| 09:57 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes1006.eqiad.wmnet |
[production] |
| 09:56 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes1015.eqiad.wmnet |
[production] |
| 09:54 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1006.eqiad.wmnet |
[production] |
| 09:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Pool db1165 into s6 with minimal weight T258361', diff saved to https://phabricator.wikimedia.org/P15029 and previous config saved to /var/cache/conftool/dbconfig/20210323-095437-marostegui.json |
[production] |
| 09:54 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 09:54 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes1016.eqiad.wmnet |
[production] |
| 09:53 |
<akosiaris> |
deploy helmfile.d/admin_ng for eqiad T277741 |
[production] |
| 09:53 |
<hashar> |
scap prep 1.36.0-wmf.36 # T274940 |
[production] |
| 09:53 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 09:53 |
<jayme@cumin1001> |
conftool action : set/pooled=yes; selector: dc=codfw,service=kubesvc,name=kubernetes2017.codfw.wmnet |
[production] |
| 09:53 |
<jayme@cumin1001> |
conftool action : set/weight=10; selector: dc=codfw,service=kubesvc,name=kubernetes2017.codfw.wmnet |
[production] |
| 09:51 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 09:50 |
<jayme@cumin1001> |
conftool action : set/pooled=yes; selector: dc=eqiad,service=kubesvc,name=kubernetes1017.eqiad.wmnet |
[production] |
| 09:50 |
<jayme@cumin1001> |
conftool action : set/weight=10; selector: dc=eqiad,service=kubesvc,name=kubernetes1017.eqiad.wmnet |
[production] |
| 09:49 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 09:46 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 09:46 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 09:45 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 09:45 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 09:45 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1014.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:44 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 09:44 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 09:43 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1014.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:43 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1013.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Pool db1165 into s6 with minimal weight T258361', diff saved to https://phabricator.wikimedia.org/P15028 and previous config saved to /var/cache/conftool/dbconfig/20210323-094257-marostegui.json |
[production] |
| 09:41 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1012.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:41 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1016.eqiad.wmnet |
[production] |
| 09:41 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1013.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:40 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1015.eqiad.wmnet |
[production] |
| 09:40 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1005.eqiad.wmnet |
[production] |
| 09:39 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1011.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:38 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1012.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:37 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1010.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:36 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1011.eqiad.wmnet with reason: REIMAGE |
[production] |
| 09:36 |
<jayme@cumin1001> |
END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) for 2:00:00 on kubernetes1004.eqiad.wmnet with reason: REIMAGE |
[production] |