|
2020-09-08
ยง
|
| 13:20 |
<elukey@cumin1001> |
START - Cookbook sre.hadoop.roll-restart-masters |
[production] |
| 13:20 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'termbox' for release 'production' . |
[production] |
| 13:20 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'termbox' for release 'test' . |
[production] |
| 13:20 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'termbox' for release 'staging' . |
[production] |
| 13:20 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'canary' . |
[production] |
| 13:20 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'production' . |
[production] |
| 13:20 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'sessionstore' for release 'production' . |
[production] |
| 13:20 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'sessionstore' for release 'staging' . |
[production] |
| 13:20 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'proton' for release 'production' . |
[production] |
| 13:18 |
<cmjohnson1> |
swapping pdu's in eqiad, mgmt for racks d3 and d4 will go down |
[production] |
| 13:18 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mobileapps' for release 'nontls' . |
[production] |
| 13:18 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mobileapps' for release 'staging' . |
[production] |
| 13:18 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mobileapps' for release 'production' . |
[production] |
| 13:17 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mathoid' for release 'staging' . |
[production] |
| 13:17 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mathoid' for release 'production' . |
[production] |
| 13:16 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-analytics' for release 'canary' . |
[production] |
| 13:16 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-analytics' for release 'production' . |
[production] |
| 13:16 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-analytics-external' for release 'canary' . |
[production] |
| 13:16 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-analytics-external' for release 'production' . |
[production] |
| 13:16 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'echostore' for release 'production' . |
[production] |
| 13:16 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'echostore' for release 'staging' . |
[production] |
| 13:14 |
<elukey@cumin1001> |
END (FAIL) - Cookbook sre.hadoop.roll-restart-masters (exit_code=99) |
[production] |
| 13:14 |
<elukey@cumin1001> |
START - Cookbook sre.hadoop.roll-restart-masters |
[production] |
| 13:13 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'changeprop-jobqueue' for release 'production' . |
[production] |
| 13:12 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'changeprop' for release 'production' . |
[production] |
| 13:09 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'cxserver' for release 'staging' . |
[production] |
| 13:09 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'cxserver' for release 'production' . |
[production] |
| 13:08 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'citoid' for release 'production' . |
[production] |
| 13:08 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'citoid' for release 'staging' . |
[production] |
| 13:04 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'blubberoid' for release 'production' . |
[production] |
| 13:04 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'blubberoid' for release 'staging' . |
[production] |
| 12:47 |
<oblivian@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'mobileapps' for release 'staging' . |
[production] |
| 12:35 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'Repooling after reboot. T261389', diff saved to https://phabricator.wikimedia.org/P12523 and previous config saved to /var/cache/conftool/dbconfig/20200908-123546-kormat.json |
[production] |
| 12:34 |
<oblivian@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'mobileapps' for release 'staging' . |
[production] |
| 12:27 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'Rebooting for T261389', diff saved to https://phabricator.wikimedia.org/P12522 and previous config saved to /var/cache/conftool/dbconfig/20200908-122702-kormat.json |
[production] |
| 12:27 |
<kormat@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 12:27 |
<kormat@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 12:11 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'Repooling after reboot. T261389', diff saved to https://phabricator.wikimedia.org/P12521 and previous config saved to /var/cache/conftool/dbconfig/20200908-121139-kormat.json |
[production] |
| 12:04 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'Rebooting for T261389', diff saved to https://phabricator.wikimedia.org/P12520 and previous config saved to /var/cache/conftool/dbconfig/20200908-120419-kormat.json |
[production] |
| 12:04 |
<kormat@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 12:04 |
<kormat@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 11:34 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'kube-system' for release 'calico-policy-controller' . |
[production] |
| 11:33 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'kube-system' for release 'coredns' . |
[production] |
| 11:33 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'kube-system' for release 'rbac-deploy-clusterrole' . |
[production] |
| 11:18 |
<jynus@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 11:15 |
<jynus@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 10:53 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'kube-system' for release 'calico-policy-controller' . |
[production] |
| 10:53 |
<marostegui> |
Deploy schema change on s3 eqiad master - T253276 |
[production] |
| 10:53 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'kube-system' for release 'coredns' . |
[production] |
| 10:53 |
<akosiaris@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'kube-system' for release 'rbac-deploy-clusterrole' . |
[production] |