|
2022-09-29
ยง
|
| 08:26 |
<elukey@deploy1002> |
helmfile [ml-serve-eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 08:22 |
<elukey@deploy1002> |
helmfile [ml-serve-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 08:21 |
<elukey@deploy1002> |
helmfile [ml-serve-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 08:15 |
<XioNoX> |
first cr1-eqiad RE switchover (for NVM firmware) - T295690 |
[production] |
| 08:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1177 (re)pooling @ 75%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35145 and previous config saved to /var/cache/conftool/dbconfig/20220929-081252-root.json |
[production] |
| 08:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2121 (re)pooling @ 100%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35144 and previous config saved to /var/cache/conftool/dbconfig/20220929-080340-root.json |
[production] |
| 07:57 |
<XioNoX> |
drain traffic away from cr1-eqiad - T295690 |
[production] |
| 07:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1177 (re)pooling @ 50%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35143 and previous config saved to /var/cache/conftool/dbconfig/20220929-075747-root.json |
[production] |
| 07:49 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on cr1-eqiad,cr1-eqiad IPv6,re0.cr1-eqiad.mgmt with reason: router upgrade |
[production] |
| 07:49 |
<ayounsi@cumin1001> |
START - Cookbook sre.hosts.downtime for 4:00:00 on cr1-eqiad,cr1-eqiad IPv6,re0.cr1-eqiad.mgmt with reason: router upgrade |
[production] |
| 07:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2121 (re)pooling @ 75%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35142 and previous config saved to /var/cache/conftool/dbconfig/20220929-074835-root.json |
[production] |
| 07:45 |
<moritzm> |
installing expat security updates |
[production] |
| 07:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1177 (re)pooling @ 25%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35141 and previous config saved to /var/cache/conftool/dbconfig/20220929-074242-root.json |
[production] |
| 07:42 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'configure' for AS: 18106 |
[production] |
| 07:40 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.peering with action 'configure' for AS: 18106 |
[production] |
| 07:38 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'configure' for AS: 38040 |
[production] |
| 07:38 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.peering with action 'configure' for AS: 38040 |
[production] |
| 07:36 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'configure' for AS: 35280 |
[production] |
| 07:34 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.peering with action 'configure' for AS: 35280 |
[production] |
| 07:33 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2121 (re)pooling @ 50%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35140 and previous config saved to /var/cache/conftool/dbconfig/20220929-073330-root.json |
[production] |
| 07:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2110 (re)pooling @ 100%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35139 and previous config saved to /var/cache/conftool/dbconfig/20220929-072745-root.json |
[production] |
| 07:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1177 (re)pooling @ 10%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35138 and previous config saved to /var/cache/conftool/dbconfig/20220929-072737-root.json |
[production] |
| 07:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2121 (re)pooling @ 25%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35137 and previous config saved to /var/cache/conftool/dbconfig/20220929-071825-root.json |
[production] |
| 07:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2110 (re)pooling @ 75%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35136 and previous config saved to /var/cache/conftool/dbconfig/20220929-071240-root.json |
[production] |
| 07:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1177 (re)pooling @ 5%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35135 and previous config saved to /var/cache/conftool/dbconfig/20220929-071232-root.json |
[production] |
| 07:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2121 (re)pooling @ 10%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35134 and previous config saved to /var/cache/conftool/dbconfig/20220929-070320-root.json |
[production] |
| 06:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2110 (re)pooling @ 50%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35133 and previous config saved to /var/cache/conftool/dbconfig/20220929-065736-root.json |
[production] |
| 06:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1177 (re)pooling @ 3%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35132 and previous config saved to /var/cache/conftool/dbconfig/20220929-065727-root.json |
[production] |
| 06:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2121 (re)pooling @ 5%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35131 and previous config saved to /var/cache/conftool/dbconfig/20220929-064815-root.json |
[production] |
| 06:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2110 (re)pooling @ 25%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35130 and previous config saved to /var/cache/conftool/dbconfig/20220929-064231-root.json |
[production] |
| 06:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1177 (re)pooling @ 1%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35129 and previous config saved to /var/cache/conftool/dbconfig/20220929-064222-root.json |
[production] |
| 06:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1177', diff saved to https://phabricator.wikimedia.org/P35128 and previous config saved to /var/cache/conftool/dbconfig/20220929-063508-root.json |
[production] |
| 06:34 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.prepare-upgrade (exit_code=0) |
[production] |
| 06:34 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.prepare-upgrade (exit_code=0) |
[production] |
| 06:33 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2121 (re)pooling @ 3%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35127 and previous config saved to /var/cache/conftool/dbconfig/20220929-063310-root.json |
[production] |
| 06:27 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.prepare-upgrade |
[production] |
| 06:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2110 (re)pooling @ 10%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35126 and previous config saved to /var/cache/conftool/dbconfig/20220929-062726-root.json |
[production] |
| 06:27 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.prepare-upgrade |
[production] |
| 06:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2121 (re)pooling @ 1%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35125 and previous config saved to /var/cache/conftool/dbconfig/20220929-061805-root.json |
[production] |
| 06:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2110 (re)pooling @ 5%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35124 and previous config saved to /var/cache/conftool/dbconfig/20220929-061221-root.json |
[production] |
| 06:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db2121 T318888', diff saved to https://phabricator.wikimedia.org/P35123 and previous config saved to /var/cache/conftool/dbconfig/20220929-060532-root.json |
[production] |
| 06:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Promote db2118 to s7 primary and set section read-write T318888', diff saved to https://phabricator.wikimedia.org/P35122 and previous config saved to /var/cache/conftool/dbconfig/20220929-060425-root.json |
[production] |
| 06:03 |
<marostegui> |
Starting s7 codfw failover from db2121 to db2118 - T318888 |
[production] |
| 05:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2110 (re)pooling @ 3%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35121 and previous config saved to /var/cache/conftool/dbconfig/20220929-055716-root.json |
[production] |
| 05:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db2118 from API T318888', diff saved to https://phabricator.wikimedia.org/P35120 and previous config saved to /var/cache/conftool/dbconfig/20220929-054542-root.json |
[production] |
| 05:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Set db2118 with weight 0 T318888', diff saved to https://phabricator.wikimedia.org/P35119 and previous config saved to /var/cache/conftool/dbconfig/20220929-054509-root.json |
[production] |
| 05:45 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 30 hosts with reason: Primary switchover s7 T318888 |
[production] |
| 05:44 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 30 hosts with reason: Primary switchover s7 T318888 |
[production] |
| 05:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2110 (re)pooling @ 1%: After upgrade', diff saved to https://phabricator.wikimedia.org/P35118 and previous config saved to /var/cache/conftool/dbconfig/20220929-054211-root.json |
[production] |
| 05:39 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db2140 from API T318886', diff saved to https://phabricator.wikimedia.org/P35117 and previous config saved to /var/cache/conftool/dbconfig/20220929-053951-root.json |
[production] |