|
2020-09-03
ยง
|
| 10:38 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'api-gateway' for release 'staging' . |
[production] |
| 10:07 |
<XioNoX> |
re-apply vlan 1118 firewall filter and update OSPF/bootp on cr1/2-eqiad - T261866 |
[production] |
| 09:57 |
<XioNoX> |
rectification: move vlan 1118 from ae2.1118 to xe-3/0/4.1118 on cr1-eqiad - T261866 |
[production] |
| 09:56 |
<XioNoX> |
move vlan 1118 from ae2.1118 to xe-3/0/4.1118 cr2-eqiad - T261866 |
[production] |
| 09:55 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Fully repool db2087:3317', diff saved to https://phabricator.wikimedia.org/P12480 and previous config saved to /var/cache/conftool/dbconfig/20200903-095510-marostegui.json |
[production] |
| 09:50 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Fully repool db2087:3316', diff saved to https://phabricator.wikimedia.org/P12479 and previous config saved to /var/cache/conftool/dbconfig/20200903-095015-marostegui.json |
[production] |
| 09:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2087:3317', diff saved to https://phabricator.wikimedia.org/P12478 and previous config saved to /var/cache/conftool/dbconfig/20200903-094857-marostegui.json |
[production] |
| 09:48 |
<XioNoX> |
move VRRP master from cr1-eqiad:ae2.1118 to cr2-eqiad:xe-3/0/4.1118 - T261866 |
[production] |
| 09:46 |
<XioNoX> |
move vlan 1118 IPv4 from ae2.1118 to xe-3/0/4.1118 cr2-eqiad - T261866 |
[production] |
| 09:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2087:3317', diff saved to https://phabricator.wikimedia.org/P12477 and previous config saved to /var/cache/conftool/dbconfig/20200903-094435-marostegui.json |
[production] |
| 09:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2087:3316', diff saved to https://phabricator.wikimedia.org/P12476 and previous config saved to /var/cache/conftool/dbconfig/20200903-094043-marostegui.json |
[production] |
| 09:38 |
<XioNoX> |
move vlan 1118 IPv6 from ae2.1118 to xe-3/0/4.1118 cr2-eqiad - T261866 |
[production] |
| 09:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2087:3317', diff saved to https://phabricator.wikimedia.org/P12475 and previous config saved to /var/cache/conftool/dbconfig/20200903-093629-marostegui.json |
[production] |
| 09:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2087:3316', diff saved to https://phabricator.wikimedia.org/P12474 and previous config saved to /var/cache/conftool/dbconfig/20200903-093454-marostegui.json |
[production] |
| 09:32 |
<aborrero@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 09:31 |
<aborrero@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 09:29 |
<hnowlan@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'api-gateway' for release 'staging' . |
[production] |
| 09:25 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2087:3316', diff saved to https://phabricator.wikimedia.org/P12473 and previous config saved to /var/cache/conftool/dbconfig/20200903-092549-marostegui.json |
[production] |
| 09:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db2087:3316 db2087:3317 T261917', diff saved to https://phabricator.wikimedia.org/P12472 and previous config saved to /var/cache/conftool/dbconfig/20200903-092028-marostegui.json |
[production] |
| 09:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Fully repool db1098:3317', diff saved to https://phabricator.wikimedia.org/P12471 and previous config saved to /var/cache/conftool/dbconfig/20200903-091834-marostegui.json |
[production] |
| 09:13 |
<XioNoX> |
rolled back: move vlan 1118 from ae2.1118 to xe-3/0/4.1118 cr2-eqiad - T261866 |
[production] |
| 09:09 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Fully repool db2122', diff saved to https://phabricator.wikimedia.org/P12470 and previous config saved to /var/cache/conftool/dbconfig/20200903-090901-marostegui.json |
[production] |
| 09:06 |
<XioNoX> |
move vlan 1118 from ae2.1118 to xe-3/0/4.1118 cr2-eqiad - T261866 |
[production] |
| 09:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Fully repool db1098:3316', diff saved to https://phabricator.wikimedia.org/P12469 and previous config saved to /var/cache/conftool/dbconfig/20200903-090419-marostegui.json |
[production] |
| 09:01 |
<XioNoX> |
force ae2.1118 VRRP master on cr1-eqiad - T261866 |
[production] |
| 09:00 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1098:3317, db1098:3316', diff saved to https://phabricator.wikimedia.org/P12468 and previous config saved to /var/cache/conftool/dbconfig/20200903-090007-marostegui.json |
[production] |
| 08:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Fully repool db1090:3317', diff saved to https://phabricator.wikimedia.org/P12467 and previous config saved to /var/cache/conftool/dbconfig/20200903-085838-marostegui.json |
[production] |
| 08:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2122', diff saved to https://phabricator.wikimedia.org/P12466 and previous config saved to /var/cache/conftool/dbconfig/20200903-085708-marostegui.json |
[production] |
| 08:49 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2122', diff saved to https://phabricator.wikimedia.org/P12465 and previous config saved to /var/cache/conftool/dbconfig/20200903-084910-marostegui.json |
[production] |
| 08:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Fully repool db1090:3312', diff saved to https://phabricator.wikimedia.org/P12464 and previous config saved to /var/cache/conftool/dbconfig/20200903-084836-marostegui.json |
[production] |
| 08:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1090:3317, db1090:3312', diff saved to https://phabricator.wikimedia.org/P12463 and previous config saved to /var/cache/conftool/dbconfig/20200903-084358-marostegui.json |
[production] |
| 08:41 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2122', diff saved to https://phabricator.wikimedia.org/P12462 and previous config saved to /var/cache/conftool/dbconfig/20200903-084147-marostegui.json |
[production] |
| 08:33 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) |
[production] |
| 08:29 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db2122 T261917', diff saved to https://phabricator.wikimedia.org/P12461 and previous config saved to /var/cache/conftool/dbconfig/20200903-082956-marostegui.json |
[production] |
| 08:28 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.reboot-single |
[production] |
| 08:28 |
<moritzm> |
rebooting mwmaint1002 for kernel update |
[production] |
| 08:26 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Fully repool db2086:3317', diff saved to https://phabricator.wikimedia.org/P12460 and previous config saved to /var/cache/conftool/dbconfig/20200903-082655-marostegui.json |
[production] |
| 08:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2086:3317', diff saved to https://phabricator.wikimedia.org/P12459 and previous config saved to /var/cache/conftool/dbconfig/20200903-082034-marostegui.json |
[production] |
| 08:16 |
<marostegui> |
Upgrade db1101 (s7 and s8) |
[production] |
| 08:15 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Fully repool db2086:3318', diff saved to https://phabricator.wikimedia.org/P12458 and previous config saved to /var/cache/conftool/dbconfig/20200903-081543-marostegui.json |
[production] |
| 08:15 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1101:3318, db1101:3317', diff saved to https://phabricator.wikimedia.org/P12457 and previous config saved to /var/cache/conftool/dbconfig/20200903-081503-marostegui.json |
[production] |
| 08:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2086:3317', diff saved to https://phabricator.wikimedia.org/P12456 and previous config saved to /var/cache/conftool/dbconfig/20200903-081337-marostegui.json |
[production] |
| 08:07 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2086:3318', diff saved to https://phabricator.wikimedia.org/P12455 and previous config saved to /var/cache/conftool/dbconfig/20200903-080714-marostegui.json |
[production] |
| 08:06 |
<marostegui> |
Upgrade and reboot db1127 |
[production] |
| 08:06 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2086:3317', diff saved to https://phabricator.wikimedia.org/P12454 and previous config saved to /var/cache/conftool/dbconfig/20200903-080634-marostegui.json |
[production] |
| 08:00 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2086:3318', diff saved to https://phabricator.wikimedia.org/P12453 and previous config saved to /var/cache/conftool/dbconfig/20200903-080024-marostegui.json |
[production] |
| 07:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Slowly repool db2086:3318', diff saved to https://phabricator.wikimedia.org/P12452 and previous config saved to /var/cache/conftool/dbconfig/20200903-075443-marostegui.json |
[production] |
| 07:49 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db2086:3318', diff saved to https://phabricator.wikimedia.org/P12451 and previous config saved to /var/cache/conftool/dbconfig/20200903-074922-marostegui.json |
[production] |
| 07:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db2086:3317 T261917', diff saved to https://phabricator.wikimedia.org/P12450 and previous config saved to /var/cache/conftool/dbconfig/20200903-074827-marostegui.json |
[production] |
| 07:45 |
<oblivian@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'mobileapps' for release 'nontls' . |
[production] |