|
2020-09-22
ยง
|
| 10:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2033 (re)pooling @ 100%: Slowly repool es2033 T261717 ', diff saved to https://phabricator.wikimedia.org/P12732 and previous config saved to /var/cache/conftool/dbconfig/20200922-101324-root.json |
[production] |
| 10:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2032 (re)pooling @ 100%: Slowly es2032 T261717 ', diff saved to https://phabricator.wikimedia.org/P12731 and previous config saved to /var/cache/conftool/dbconfig/20200922-101308-root.json |
[production] |
| 10:00 |
<kormat> |
deploying schema change to s2 in eqiad. labsdb will have s2 lag until this finishes. T259831 |
[production] |
| 09:59 |
<jayme> |
running ipvsadm -D -t 10.2.1.45:34192; ipvsadm -D -t 10.2.1.42:35192 on lvs2010.codfw.wmnet,lvs2009.codfw.wmnet - T255873 T255870 |
[production] |
| 09:59 |
<jayme> |
running ipvsadm -D -t 10.2.2.45:34192; ipvsadm -D -t 10.2.2.42:35192 on lvs1016.eqiad.wmnet,lvs1015.eqiad.wmnet - T255873 T255870 |
[production] |
| 09:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2034 (re)pooling @ 75%: Slowly repool es2034 T261717 ', diff saved to https://phabricator.wikimedia.org/P12730 and previous config saved to /var/cache/conftool/dbconfig/20200922-095839-root.json |
[production] |
| 09:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2033 (re)pooling @ 75%: Slowly repool es2033 T261717 ', diff saved to https://phabricator.wikimedia.org/P12729 and previous config saved to /var/cache/conftool/dbconfig/20200922-095821-root.json |
[production] |
| 09:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2032 (re)pooling @ 75%: Slowly es2032 T261717 ', diff saved to https://phabricator.wikimedia.org/P12728 and previous config saved to /var/cache/conftool/dbconfig/20200922-095805-root.json |
[production] |
| 09:57 |
<jayme> |
restarting pybal on lvs1015.eqiad.wmnet,lvs2009.codfw.wmnet - T255873 T255870 |
[production] |
| 09:55 |
<jayme> |
restarting pybal on lvs1016.eqiad.wmnet,lvs2010.codfw.wmnet - T255873 T255870 |
[production] |
| 09:51 |
<jayme> |
running puppet on lvs servers - T255873 T255870 |
[production] |
| 09:46 |
<jbond@cumin1001> |
END (FAIL) - Cookbook sre.pdus.rotate-password (exit_code=99) |
[production] |
| 09:46 |
<jbond@cumin1001> |
START - Cookbook sre.pdus.rotate-password |
[production] |
| 09:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2034 (re)pooling @ 50%: Slowly repool es2034 T261717 ', diff saved to https://phabricator.wikimedia.org/P12727 and previous config saved to /var/cache/conftool/dbconfig/20200922-094336-root.json |
[production] |
| 09:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2033 (re)pooling @ 50%: Slowly repool es2033 T261717 ', diff saved to https://phabricator.wikimedia.org/P12726 and previous config saved to /var/cache/conftool/dbconfig/20200922-094317-root.json |
[production] |
| 09:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2032 (re)pooling @ 50%: Slowly es2032 T261717 ', diff saved to https://phabricator.wikimedia.org/P12725 and previous config saved to /var/cache/conftool/dbconfig/20200922-094302-root.json |
[production] |
| 09:30 |
<volans> |
repooling ulsfo after merging DNS migration to Netbox zonefiles - T258729 |
[production] |
| 09:30 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.pdus.uptime (exit_code=0) |
[production] |
| 09:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2034 (re)pooling @ 25%: Slowly repool es2034 T261717 ', diff saved to https://phabricator.wikimedia.org/P12724 and previous config saved to /var/cache/conftool/dbconfig/20200922-092832-root.json |
[production] |
| 09:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2033 (re)pooling @ 25%: Slowly repool es2033 T261717 ', diff saved to https://phabricator.wikimedia.org/P12723 and previous config saved to /var/cache/conftool/dbconfig/20200922-092814-root.json |
[production] |
| 09:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2032 (re)pooling @ 25%: Slowly es2032 T261717 ', diff saved to https://phabricator.wikimedia.org/P12722 and previous config saved to /var/cache/conftool/dbconfig/20200922-092758-root.json |
[production] |
| 09:26 |
<jbond@cumin1001> |
START - Cookbook sre.pdus.uptime |
[production] |
| 09:24 |
<XioNoX> |
replace BGP_IXP_in with BGP_IXP_PRIMARY_in on cr3-ulsfo IX BGP group - T262517 |
[production] |
| 09:22 |
<XioNoX> |
add BGP_IXP_PRIMARY_in to cr3-ulsfo - T262517 |
[production] |
| 09:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2034 (re)pooling @ 10%: Slowly repool es2034 T261717 ', diff saved to https://phabricator.wikimedia.org/P12721 and previous config saved to /var/cache/conftool/dbconfig/20200922-091329-root.json |
[production] |
| 09:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2033 (re)pooling @ 10%: Slowly repool es2033 T261717 ', diff saved to https://phabricator.wikimedia.org/P12720 and previous config saved to /var/cache/conftool/dbconfig/20200922-091310-root.json |
[production] |
| 09:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es2032 (re)pooling @ 10%: Slowly es2032 T261717 ', diff saved to https://phabricator.wikimedia.org/P12719 and previous config saved to /var/cache/conftool/dbconfig/20200922-091255-root.json |
[production] |
| 09:11 |
<jbond42> |
update snmp string on ps1-a8-codfw |
[production] |
| 09:05 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db2076 (re)pooling @ 100%: schema change T259831', diff saved to https://phabricator.wikimedia.org/P12718 and previous config saved to /var/cache/conftool/dbconfig/20200922-090520-kormat.json |
[production] |
| 08:58 |
<_joe_> |
restart pybal on lvs2009 |
[production] |
| 08:56 |
<_joe_> |
restarting pybal on lvs2010 |
[production] |
| 08:54 |
<_joe_> |
restarted pybal on lvs1015 |
[production] |
| 08:50 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db2076 (re)pooling @ 75%: schema change T259831', diff saved to https://phabricator.wikimedia.org/P12717 and previous config saved to /var/cache/conftool/dbconfig/20200922-085017-kormat.json |
[production] |
| 08:36 |
<_joe_> |
restarting pybal low-traffic in eqiad to pick up lvs changes |
[production] |
| 08:35 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db2076 (re)pooling @ 50%: schema change T259831', diff saved to https://phabricator.wikimedia.org/P12715 and previous config saved to /var/cache/conftool/dbconfig/20200922-083514-kormat.json |
[production] |
| 08:22 |
<volans> |
migrating ulsfo public DNS records to the Netbox-generated ones - T258729 |
[production] |
| 08:20 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db2076 (re)pooling @ 25%: schema change T259831', diff saved to https://phabricator.wikimedia.org/P12714 and previous config saved to /var/cache/conftool/dbconfig/20200922-082010-kormat.json |
[production] |
| 08:13 |
<kormat> |
uploaded wmfmariadbpy v0.5 to apt. deploying now to fleet |
[production] |
| 08:11 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Pool es2032, es2033 and es2034 for the first time with minimal weight T261717', diff saved to https://phabricator.wikimedia.org/P12713 and previous config saved to /var/cache/conftool/dbconfig/20200922-081154-marostegui.json |
[production] |
| 07:57 |
<volans> |
migrating ulsfo private DNS records to the Netbox-generated ones - T258729 |
[production] |
| 07:54 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'db2076 depooling: schema change T259831', diff saved to https://phabricator.wikimedia.org/P12712 and previous config saved to /var/cache/conftool/dbconfig/20200922-075429-kormat.json |
[production] |
| 07:51 |
<jayme> |
running ipvsadm -D -t 10.2.1.18:8080; ipvsadm -D -t 10.2.1.46:3030 on lvs2010.codfw.wmnet,lvs2009.codfw.wmnet - T255879 T254581 |
[production] |
| 07:49 |
<jayme> |
running ipvsadm -D -t 10.2.2.18:8080; ipvsadm -D -t 10.2.2.46:3030 on lvs1016.eqiad.wmnet,lvs1015.eqiad.wmnet - T255879 T254581 |
[production] |
| 07:46 |
<jayme> |
restarting pybal on lvs1015.eqiad.wmnet,lvs2009.codfw.wmnet - T255879 T254581 |
[production] |
| 07:42 |
<jayme> |
restarting pybal on lvs1016.eqiad.wmnet,lvs2010.codfw.wmnet - T255879 T254581 |
[production] |
| 07:39 |
<jayme> |
running puppet on lvs servers - T255879 T254581 |
[production] |
| 07:34 |
<volans> |
depooling ulsfo to merge DNS migration to Netbox zonefiles - T258729 |
[production] |
| 07:24 |
<marostegui> |
Stop MySQL on es2014 - host will be decommissioned T262889 |
[production] |
| 07:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Remove es2014 from dbctl T262889', diff saved to https://phabricator.wikimedia.org/P12711 and previous config saved to /var/cache/conftool/dbconfig/20200922-071435-marostegui.json |
[production] |
| 07:11 |
<XioNoX> |
cr1-codfw# run clear bfd session address fe80::f27c:c7ff:fe11:2c1b |
[production] |