|
2022-09-08
ยง
|
| 07:39 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1203 (re)pooling @ 10%: Pooling for the first time in s8', diff saved to https://phabricator.wikimedia.org/P34161 and previous config saved to /var/cache/conftool/dbconfig/20220908-073935-root.json |
[production] |
| 07:39 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2028 (re)pooling @ 4%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34160 and previous config saved to /var/cache/conftool/dbconfig/20220908-073900-root.json |
[production] |
| 07:38 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2027 (re)pooling @ 4%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34159 and previous config saved to /var/cache/conftool/dbconfig/20220908-073851-root.json |
[production] |
| 07:38 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2026 (re)pooling @ 4%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34158 and previous config saved to /var/cache/conftool/dbconfig/20220908-073846-root.json |
[production] |
| 07:37 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1202 (re)pooling @ 50%: Pooling for the first time in s7', diff saved to https://phabricator.wikimedia.org/P34157 and previous config saved to /var/cache/conftool/dbconfig/20220908-073724-root.json |
[production] |
| 07:24 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1203 (re)pooling @ 5%: Pooling for the first time in s8', diff saved to https://phabricator.wikimedia.org/P34156 and previous config saved to /var/cache/conftool/dbconfig/20220908-072405-root.json |
[production] |
| 07:23 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2028 (re)pooling @ 3%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34155 and previous config saved to /var/cache/conftool/dbconfig/20220908-072330-root.json |
[production] |
| 07:23 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2027 (re)pooling @ 3%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34154 and previous config saved to /var/cache/conftool/dbconfig/20220908-072321-root.json |
[production] |
| 07:23 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2026 (re)pooling @ 3%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34153 and previous config saved to /var/cache/conftool/dbconfig/20220908-072316-root.json |
[production] |
| 07:21 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1202 (re)pooling @ 25%: Pooling for the first time in s7', diff saved to https://phabricator.wikimedia.org/P34152 and previous config saved to /var/cache/conftool/dbconfig/20220908-072154-root.json |
[production] |
| 07:08 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1203 (re)pooling @ 4%: Pooling for the first time in s8', diff saved to https://phabricator.wikimedia.org/P34151 and previous config saved to /var/cache/conftool/dbconfig/20220908-070836-root.json |
[production] |
| 07:08 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2028 (re)pooling @ 2%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34150 and previous config saved to /var/cache/conftool/dbconfig/20220908-070800-root.json |
[production] |
| 07:07 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2027 (re)pooling @ 2%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34149 and previous config saved to /var/cache/conftool/dbconfig/20220908-070752-root.json |
[production] |
| 07:07 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2026 (re)pooling @ 2%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34148 and previous config saved to /var/cache/conftool/dbconfig/20220908-070746-root.json |
[production] |
| 07:06 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1202 (re)pooling @ 10%: Pooling for the first time in s7', diff saved to https://phabricator.wikimedia.org/P34147 and previous config saved to /var/cache/conftool/dbconfig/20220908-070625-root.json |
[production] |
| 07:01 |
<elukey@deploy1002> |
helmfile [ml-serve-eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 07:01 |
<elukey@deploy1002> |
helmfile [ml-serve-eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 07:01 |
<elukey@deploy1002> |
helmfile [ml-serve-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 07:00 |
<elukey@deploy1002> |
helmfile [ml-serve-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 07:00 |
<elukey@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 07:00 |
<elukey@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 06:53 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1203 (re)pooling @ 3%: Pooling for the first time in s8', diff saved to https://phabricator.wikimedia.org/P34146 and previous config saved to /var/cache/conftool/dbconfig/20220908-065306-root.json |
[production] |
| 06:52 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2028 (re)pooling @ 1%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34145 and previous config saved to /var/cache/conftool/dbconfig/20220908-065229-root.json |
[production] |
| 06:52 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2027 (re)pooling @ 1%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34144 and previous config saved to /var/cache/conftool/dbconfig/20220908-065222-root.json |
[production] |
| 06:52 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'es2026 (re)pooling @ 1%: Repooling after upgrade', diff saved to https://phabricator.wikimedia.org/P34143 and previous config saved to /var/cache/conftool/dbconfig/20220908-065216-root.json |
[production] |
| 06:50 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1202 (re)pooling @ 5%: Pooling for the first time in s7', diff saved to https://phabricator.wikimedia.org/P34142 and previous config saved to /var/cache/conftool/dbconfig/20220908-065054-root.json |
[production] |
| 06:44 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'Depool es2026, es2027, es2028', diff saved to https://phabricator.wikimedia.org/P34141 and previous config saved to /var/cache/conftool/dbconfig/20220908-064450-marostegui.json |
[production] |
| 06:37 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1203 (re)pooling @ 2%: Pooling for the first time in s8', diff saved to https://phabricator.wikimedia.org/P34140 and previous config saved to /var/cache/conftool/dbconfig/20220908-063737-root.json |
[production] |
| 06:35 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1202 (re)pooling @ 4%: Pooling for the first time in s7', diff saved to https://phabricator.wikimedia.org/P34139 and previous config saved to /var/cache/conftool/dbconfig/20220908-063525-root.json |
[production] |
| 06:19 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1202 (re)pooling @ 3%: Pooling for the first time in s7', diff saved to https://phabricator.wikimedia.org/P34138 and previous config saved to /var/cache/conftool/dbconfig/20220908-061955-root.json |
[production] |
| 06:14 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'Add db1203 to s8, depooled, T316342', diff saved to https://phabricator.wikimedia.org/P34137 and previous config saved to /var/cache/conftool/dbconfig/20220908-061413-marostegui.json |
[production] |
| 06:09 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1157.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:09 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1157.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:04 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depool db1157 T316622', diff saved to https://phabricator.wikimedia.org/P34136 and previous config saved to /var/cache/conftool/dbconfig/20220908-060438-ladsgroup.json |
[production] |
| 06:04 |
<marostegui@cumin2002> |
dbctl commit (dc=all): 'db1202 (re)pooling @ 2%: Pooling for the first time in s7', diff saved to https://phabricator.wikimedia.org/P34135 and previous config saved to /var/cache/conftool/dbconfig/20220908-060426-root.json |
[production] |
| 06:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Promote db1123 to s3 primary and set section read-write T316622', diff saved to https://phabricator.wikimedia.org/P34134 and previous config saved to /var/cache/conftool/dbconfig/20220908-060138-ladsgroup.json |
[production] |
| 06:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set s3 eqiad as read-only for maintenance - T316622', diff saved to https://phabricator.wikimedia.org/P34133 and previous config saved to /var/cache/conftool/dbconfig/20220908-060110-ladsgroup.json |
[production] |
| 06:00 |
<Amir1> |
Starting s3 eqiad failover from db1157 to db1123 - T316622 |
[production] |
| 05:55 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Increase weight for db1194', diff saved to https://phabricator.wikimedia.org/P34132 and previous config saved to /var/cache/conftool/dbconfig/20220908-055546-marostegui.json |
[production] |
| 05:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Pooling db1202 for the first time in s7 T316342', diff saved to https://phabricator.wikimedia.org/P34131 and previous config saved to /var/cache/conftool/dbconfig/20220908-055451-marostegui.json |
[production] |
| 05:49 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Pooling back db2140', diff saved to https://phabricator.wikimedia.org/P34130 and previous config saved to /var/cache/conftool/dbconfig/20220908-054921-ladsgroup.json |
[production] |
| 05:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Add db1202 to s7, depooled, T316342', diff saved to https://phabricator.wikimedia.org/P34129 and previous config saved to /var/cache/conftool/dbconfig/20220908-054429-marostegui.json |
[production] |
| 05:10 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set db1123 with weight 0 T316622', diff saved to https://phabricator.wikimedia.org/P34128 and previous config saved to /var/cache/conftool/dbconfig/20220908-051043-ladsgroup.json |
[production] |
| 05:10 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 23 hosts with reason: Primary switchover s3 T316622 |
[production] |
| 05:09 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 23 hosts with reason: Primary switchover s3 T316622 |
[production] |
| 02:04 |
<pt1979@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host kafka-logging1005: |
[production] |
| 02:04 |
<pt1979@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host kafka-logging1005: |
[production] |
| 02:02 |
<pt1979@cumin1001> |
END (ERROR) - Cookbook sre.network.configure-switch-interfaces (exit_code=97) for host db2169 |
[production] |
| 02:02 |
<pt1979@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host db2169 |
[production] |
| 02:01 |
<pt1979@cumin1001> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |