|
2022-11-24
ยง
|
| 07:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set s7 eqiad as read-only for maintenance - T323117', diff saved to https://phabricator.wikimedia.org/P40931 and previous config saved to /var/cache/conftool/dbconfig/20221124-070215-ladsgroup.json |
[production] |
| 07:02 |
<Amir1> |
Starting s7 eqiad failover from db1181 to db1136 - T323117 |
[production] |
| 06:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40930 and previous config saved to /var/cache/conftool/dbconfig/20221124-065956-ladsgroup.json |
[production] |
| 06:56 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db2118.codfw.wmnet with reason: Maintenance |
[production] |
| 06:56 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db2118.codfw.wmnet with reason: Maintenance |
[production] |
| 06:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1187 (T323214)', diff saved to https://phabricator.wikimedia.org/P40929 and previous config saved to /var/cache/conftool/dbconfig/20221124-065057-ladsgroup.json |
[production] |
| 06:07 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set db1136 with weight 0 T323117', diff saved to https://phabricator.wikimedia.org/P40928 and previous config saved to /var/cache/conftool/dbconfig/20221124-060742-ladsgroup.json |
[production] |
| 06:06 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 30 hosts with reason: Primary switchover s7 T323117 |
[production] |
| 06:06 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 30 hosts with reason: Primary switchover s7 T323117 |
[production] |
| 06:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1187 (T323214)', diff saved to https://phabricator.wikimedia.org/P40927 and previous config saved to /var/cache/conftool/dbconfig/20221124-060330-ladsgroup.json |
[production] |
| 06:03 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1187.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:03 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1187.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40926 and previous config saved to /var/cache/conftool/dbconfig/20221124-060309-ladsgroup.json |
[production] |
| 05:48 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P40925 and previous config saved to /var/cache/conftool/dbconfig/20221124-054802-ladsgroup.json |
[production] |
| 05:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P40924 and previous config saved to /var/cache/conftool/dbconfig/20221124-053256-ladsgroup.json |
[production] |
| 05:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40923 and previous config saved to /var/cache/conftool/dbconfig/20221124-052830-ladsgroup.json |
[production] |
| 05:28 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2180.codfw.wmnet with reason: Maintenance |
[production] |
| 05:28 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2180.codfw.wmnet with reason: Maintenance |
[production] |
| 05:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2171:3316 (T323214)', diff saved to https://phabricator.wikimedia.org/P40922 and previous config saved to /var/cache/conftool/dbconfig/20221124-052808-ladsgroup.json |
[production] |
| 05:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40921 and previous config saved to /var/cache/conftool/dbconfig/20221124-051749-ladsgroup.json |
[production] |
| 05:13 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2171:3316', diff saved to https://phabricator.wikimedia.org/P40920 and previous config saved to /var/cache/conftool/dbconfig/20221124-051301-ladsgroup.json |
[production] |
| 04:57 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2171:3316', diff saved to https://phabricator.wikimedia.org/P40919 and previous config saved to /var/cache/conftool/dbconfig/20221124-045755-ladsgroup.json |
[production] |
| 04:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2171:3316 (T323214)', diff saved to https://phabricator.wikimedia.org/P40918 and previous config saved to /var/cache/conftool/dbconfig/20221124-044249-ladsgroup.json |
[production] |
| 04:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40917 and previous config saved to /var/cache/conftool/dbconfig/20221124-042757-ladsgroup.json |
[production] |
| 04:27 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1180.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:27 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1180.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T323214)', diff saved to https://phabricator.wikimedia.org/P40916 and previous config saved to /var/cache/conftool/dbconfig/20221124-042736-ladsgroup.json |
[production] |
| 04:12 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P40915 and previous config saved to /var/cache/conftool/dbconfig/20221124-041230-ladsgroup.json |
[production] |
| 03:57 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P40914 and previous config saved to /var/cache/conftool/dbconfig/20221124-035723-ladsgroup.json |
[production] |
| 03:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T323214)', diff saved to https://phabricator.wikimedia.org/P40913 and previous config saved to /var/cache/conftool/dbconfig/20221124-034217-ladsgroup.json |
[production] |
| 03:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2171:3316 (T323214)', diff saved to https://phabricator.wikimedia.org/P40912 and previous config saved to /var/cache/conftool/dbconfig/20221124-030901-ladsgroup.json |
[production] |
| 03:08 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2171.codfw.wmnet with reason: Maintenance |
[production] |
| 03:08 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2171.codfw.wmnet with reason: Maintenance |
[production] |
| 03:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3316 (T323214)', diff saved to https://phabricator.wikimedia.org/P40911 and previous config saved to /var/cache/conftool/dbconfig/20221124-030829-ladsgroup.json |
[production] |
| 03:00 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2182 (T321126)', diff saved to https://phabricator.wikimedia.org/P40910 and previous config saved to /var/cache/conftool/dbconfig/20221124-030025-marostegui.json |
[production] |
| 02:53 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3316', diff saved to https://phabricator.wikimedia.org/P40909 and previous config saved to /var/cache/conftool/dbconfig/20221124-025322-ladsgroup.json |
[production] |
| 02:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2182', diff saved to https://phabricator.wikimedia.org/P40908 and previous config saved to /var/cache/conftool/dbconfig/20221124-024518-marostegui.json |
[production] |
| 02:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3316', diff saved to https://phabricator.wikimedia.org/P40907 and previous config saved to /var/cache/conftool/dbconfig/20221124-023816-ladsgroup.json |
[production] |
| 02:35 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1168 (T323214)', diff saved to https://phabricator.wikimedia.org/P40906 and previous config saved to /var/cache/conftool/dbconfig/20221124-023500-ladsgroup.json |
[production] |
| 02:34 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1168.eqiad.wmnet with reason: Maintenance |
[production] |
| 02:34 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1168.eqiad.wmnet with reason: Maintenance |
[production] |
| 02:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1165 (T323214)', diff saved to https://phabricator.wikimedia.org/P40905 and previous config saved to /var/cache/conftool/dbconfig/20221124-023428-ladsgroup.json |
[production] |
| 02:30 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2182', diff saved to https://phabricator.wikimedia.org/P40904 and previous config saved to /var/cache/conftool/dbconfig/20221124-023011-marostegui.json |
[production] |
| 02:23 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3316 (T323214)', diff saved to https://phabricator.wikimedia.org/P40903 and previous config saved to /var/cache/conftool/dbconfig/20221124-022309-ladsgroup.json |
[production] |
| 02:19 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1165', diff saved to https://phabricator.wikimedia.org/P40902 and previous config saved to /var/cache/conftool/dbconfig/20221124-021921-ladsgroup.json |
[production] |
| 02:15 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2182 (T321126)', diff saved to https://phabricator.wikimedia.org/P40901 and previous config saved to /var/cache/conftool/dbconfig/20221124-021505-marostegui.json |
[production] |
| 02:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2182 (T321126)', diff saved to https://phabricator.wikimedia.org/P40900 and previous config saved to /var/cache/conftool/dbconfig/20221124-021233-marostegui.json |
[production] |
| 02:12 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db2182.codfw.wmnet with reason: Maintenance |
[production] |
| 02:12 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db2182.codfw.wmnet with reason: Maintenance |
[production] |
| 02:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317 (T321126)', diff saved to https://phabricator.wikimedia.org/P40899 and previous config saved to /var/cache/conftool/dbconfig/20221124-021211-marostegui.json |
[production] |