|
2022-04-21
ยง
|
| 07:02 |
<jynus@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on backup2006.codfw.wmnet with reason: host reimage |
[production] |
| 07:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1113:3315 (T298565)', diff saved to https://phabricator.wikimedia.org/P25899 and previous config saved to /var/cache/conftool/dbconfig/20220421-070208-ladsgroup.json |
[production] |
| 07:02 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:02 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161 (T298565)', diff saved to https://phabricator.wikimedia.org/P25898 and previous config saved to /var/cache/conftool/dbconfig/20220421-070113-ladsgroup.json |
[production] |
| 07:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3317', diff saved to https://phabricator.wikimedia.org/P25897 and previous config saved to /var/cache/conftool/dbconfig/20220421-070019-ladsgroup.json |
[production] |
| 06:52 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1166', diff saved to https://phabricator.wikimedia.org/P25896 and previous config saved to /var/cache/conftool/dbconfig/20220421-065211-ladsgroup.json |
[production] |
| 06:46 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161', diff saved to https://phabricator.wikimedia.org/P25895 and previous config saved to /var/cache/conftool/dbconfig/20220421-064608-ladsgroup.json |
[production] |
| 06:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3317 (T306560)', diff saved to https://phabricator.wikimedia.org/P25894 and previous config saved to /var/cache/conftool/dbconfig/20220421-064514-ladsgroup.json |
[production] |
| 06:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1170:3317 (T306560)', diff saved to https://phabricator.wikimedia.org/P25893 and previous config saved to /var/cache/conftool/dbconfig/20220421-064245-ladsgroup.json |
[production] |
| 06:42 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:42 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:42 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:42 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136 (T306560)', diff saved to https://phabricator.wikimedia.org/P25892 and previous config saved to /var/cache/conftool/dbconfig/20220421-064210-ladsgroup.json |
[production] |
| 06:37 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1166', diff saved to https://phabricator.wikimedia.org/P25891 and previous config saved to /var/cache/conftool/dbconfig/20220421-063706-ladsgroup.json |
[production] |
| 06:34 |
<jynus@cumin2002> |
START - Cookbook sre.hosts.reimage for host backup2006.codfw.wmnet with OS bullseye |
[production] |
| 06:31 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161', diff saved to https://phabricator.wikimedia.org/P25890 and previous config saved to /var/cache/conftool/dbconfig/20220421-063103-ladsgroup.json |
[production] |
| 06:30 |
<jynus@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host backup1006.eqiad.wmnet with OS bullseye |
[production] |
| 06:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136', diff saved to https://phabricator.wikimedia.org/P25889 and previous config saved to /var/cache/conftool/dbconfig/20220421-062705-ladsgroup.json |
[production] |
| 06:22 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1166 (T298565)', diff saved to https://phabricator.wikimedia.org/P25888 and previous config saved to /var/cache/conftool/dbconfig/20220421-062201-ladsgroup.json |
[production] |
| 06:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161 (T298565)', diff saved to https://phabricator.wikimedia.org/P25887 and previous config saved to /var/cache/conftool/dbconfig/20220421-061558-ladsgroup.json |
[production] |
| 06:12 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136', diff saved to https://phabricator.wikimedia.org/P25886 and previous config saved to /var/cache/conftool/dbconfig/20220421-061200-ladsgroup.json |
[production] |
| 06:11 |
<jynus@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on backup1006.eqiad.wmnet with reason: host reimage |
[production] |
| 06:08 |
<jynus@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on backup1006.eqiad.wmnet with reason: host reimage |
[production] |
| 06:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1109 T303927', diff saved to https://phabricator.wikimedia.org/P25885 and previous config saved to /var/cache/conftool/dbconfig/20220421-060512-root.json |
[production] |
| 06:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Promote db1104 to s8 primary and set section read-write T303927', diff saved to https://phabricator.wikimedia.org/P25884 and previous config saved to /var/cache/conftool/dbconfig/20220421-060106-ladsgroup.json |
[production] |
| 06:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set s8 eqiad as read-only for maintenance - T303927', diff saved to https://phabricator.wikimedia.org/P25883 and previous config saved to /var/cache/conftool/dbconfig/20220421-060023-ladsgroup.json |
[production] |
| 06:00 |
<Amir1> |
Starting s8 eqiad failover from db1109 to db1104 - T303927 |
[production] |
| 05:57 |
<jynus@cumin1001> |
START - Cookbook sre.hosts.reimage for host backup1006.eqiad.wmnet with OS bullseye |
[production] |
| 05:56 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136 (T306560)', diff saved to https://phabricator.wikimedia.org/P25882 and previous config saved to /var/cache/conftool/dbconfig/20220421-055655-ladsgroup.json |
[production] |
| 05:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1136 (T306560)', diff saved to https://phabricator.wikimedia.org/P25881 and previous config saved to /var/cache/conftool/dbconfig/20220421-055441-ladsgroup.json |
[production] |
| 05:54 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:54 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127 (T306560)', diff saved to https://phabricator.wikimedia.org/P25880 and previous config saved to /var/cache/conftool/dbconfig/20220421-055433-ladsgroup.json |
[production] |
| 05:39 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127', diff saved to https://phabricator.wikimedia.org/P25879 and previous config saved to /var/cache/conftool/dbconfig/20220421-053928-ladsgroup.json |
[production] |
| 05:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127', diff saved to https://phabricator.wikimedia.org/P25878 and previous config saved to /var/cache/conftool/dbconfig/20220421-052423-ladsgroup.json |
[production] |
| 05:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1166 (T298565)', diff saved to https://phabricator.wikimedia.org/P25877 and previous config saved to /var/cache/conftool/dbconfig/20220421-052146-ladsgroup.json |
[production] |
| 05:21 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1166.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:21 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1166.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1161 (T298565)', diff saved to https://phabricator.wikimedia.org/P25876 and previous config saved to /var/cache/conftool/dbconfig/20220421-051543-ladsgroup.json |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on clouddb[1016,1020-1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on clouddb[1016,1020-1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1161.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1161.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3315 (T298565)', diff saved to https://phabricator.wikimedia.org/P25875 and previous config saved to /var/cache/conftool/dbconfig/20220421-051529-ladsgroup.json |
[production] |
| 05:09 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1132 T301879', diff saved to https://phabricator.wikimedia.org/P25874 and previous config saved to /var/cache/conftool/dbconfig/20220421-050931-marostegui.json |
[production] |
| 05:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127 (T306560)', diff saved to https://phabricator.wikimedia.org/P25873 and previous config saved to /var/cache/conftool/dbconfig/20220421-050918-ladsgroup.json |
[production] |
| 05:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set db1104 with weight 0 T303927', diff saved to https://phabricator.wikimedia.org/P25872 and previous config saved to /var/cache/conftool/dbconfig/20220421-050154-ladsgroup.json |
[production] |
| 05:01 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on 31 hosts with reason: Primary switchover s8 T303927 |
[production] |