|
2022-04-21
ยง
|
| 06:08 |
<jynus@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on backup1006.eqiad.wmnet with reason: host reimage |
[production] |
| 06:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1109 T303927', diff saved to https://phabricator.wikimedia.org/P25885 and previous config saved to /var/cache/conftool/dbconfig/20220421-060512-root.json |
[production] |
| 06:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Promote db1104 to s8 primary and set section read-write T303927', diff saved to https://phabricator.wikimedia.org/P25884 and previous config saved to /var/cache/conftool/dbconfig/20220421-060106-ladsgroup.json |
[production] |
| 06:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set s8 eqiad as read-only for maintenance - T303927', diff saved to https://phabricator.wikimedia.org/P25883 and previous config saved to /var/cache/conftool/dbconfig/20220421-060023-ladsgroup.json |
[production] |
| 06:00 |
<Amir1> |
Starting s8 eqiad failover from db1109 to db1104 - T303927 |
[production] |
| 05:57 |
<jynus@cumin1001> |
START - Cookbook sre.hosts.reimage for host backup1006.eqiad.wmnet with OS bullseye |
[production] |
| 05:56 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136 (T306560)', diff saved to https://phabricator.wikimedia.org/P25882 and previous config saved to /var/cache/conftool/dbconfig/20220421-055655-ladsgroup.json |
[production] |
| 05:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1136 (T306560)', diff saved to https://phabricator.wikimedia.org/P25881 and previous config saved to /var/cache/conftool/dbconfig/20220421-055441-ladsgroup.json |
[production] |
| 05:54 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:54 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127 (T306560)', diff saved to https://phabricator.wikimedia.org/P25880 and previous config saved to /var/cache/conftool/dbconfig/20220421-055433-ladsgroup.json |
[production] |
| 05:39 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127', diff saved to https://phabricator.wikimedia.org/P25879 and previous config saved to /var/cache/conftool/dbconfig/20220421-053928-ladsgroup.json |
[production] |
| 05:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127', diff saved to https://phabricator.wikimedia.org/P25878 and previous config saved to /var/cache/conftool/dbconfig/20220421-052423-ladsgroup.json |
[production] |
| 05:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1166 (T298565)', diff saved to https://phabricator.wikimedia.org/P25877 and previous config saved to /var/cache/conftool/dbconfig/20220421-052146-ladsgroup.json |
[production] |
| 05:21 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1166.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:21 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1166.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1161 (T298565)', diff saved to https://phabricator.wikimedia.org/P25876 and previous config saved to /var/cache/conftool/dbconfig/20220421-051543-ladsgroup.json |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on clouddb[1016,1020-1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on clouddb[1016,1020-1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1161.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1161.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3315 (T298565)', diff saved to https://phabricator.wikimedia.org/P25875 and previous config saved to /var/cache/conftool/dbconfig/20220421-051529-ladsgroup.json |
[production] |
| 05:09 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1132 T301879', diff saved to https://phabricator.wikimedia.org/P25874 and previous config saved to /var/cache/conftool/dbconfig/20220421-050931-marostegui.json |
[production] |
| 05:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127 (T306560)', diff saved to https://phabricator.wikimedia.org/P25873 and previous config saved to /var/cache/conftool/dbconfig/20220421-050918-ladsgroup.json |
[production] |
| 05:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set db1104 with weight 0 T303927', diff saved to https://phabricator.wikimedia.org/P25872 and previous config saved to /var/cache/conftool/dbconfig/20220421-050154-ladsgroup.json |
[production] |
| 05:01 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on 31 hosts with reason: Primary switchover s8 T303927 |
[production] |
| 05:01 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on 31 hosts with reason: Primary switchover s8 T303927 |
[production] |
| 05:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3315', diff saved to https://phabricator.wikimedia.org/P25871 and previous config saved to /var/cache/conftool/dbconfig/20220421-050024-ladsgroup.json |
[production] |
| 04:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3315', diff saved to https://phabricator.wikimedia.org/P25870 and previous config saved to /var/cache/conftool/dbconfig/20220421-044519-ladsgroup.json |
[production] |
| 04:37 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:37 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:30 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3315 (T298565)', diff saved to https://phabricator.wikimedia.org/P25869 and previous config saved to /var/cache/conftool/dbconfig/20220421-043014-ladsgroup.json |
[production] |
| 04:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1127 (T306560)', diff saved to https://phabricator.wikimedia.org/P25868 and previous config saved to /var/cache/conftool/dbconfig/20220421-042545-ladsgroup.json |
[production] |
| 04:25 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1127.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:25 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1127.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1174 (T306560)', diff saved to https://phabricator.wikimedia.org/P25867 and previous config saved to /var/cache/conftool/dbconfig/20220421-042537-ladsgroup.json |
[production] |
| 04:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1113:3315 (T298565)', diff saved to https://phabricator.wikimedia.org/P25866 and previous config saved to /var/cache/conftool/dbconfig/20220421-042142-ladsgroup.json |
[production] |
| 04:21 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:21 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:17 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1096.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:17 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1096.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1144:3315 (T298565)', diff saved to https://phabricator.wikimedia.org/P25865 and previous config saved to /var/cache/conftool/dbconfig/20220421-041710-ladsgroup.json |
[production] |
| 04:10 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1174', diff saved to https://phabricator.wikimedia.org/P25864 and previous config saved to /var/cache/conftool/dbconfig/20220421-041032-ladsgroup.json |
[production] |
| 04:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1144:3315', diff saved to https://phabricator.wikimedia.org/P25863 and previous config saved to /var/cache/conftool/dbconfig/20220421-040204-ladsgroup.json |
[production] |
| 03:55 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1174', diff saved to https://phabricator.wikimedia.org/P25862 and previous config saved to /var/cache/conftool/dbconfig/20220421-035526-ladsgroup.json |
[production] |
| 03:47 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1144:3315', diff saved to https://phabricator.wikimedia.org/P25861 and previous config saved to /var/cache/conftool/dbconfig/20220421-034659-ladsgroup.json |
[production] |
| 03:44 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db[2074,2094,2109,2127,2149].codfw.wmnet with reason: Maintenance |
[production] |
| 03:44 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db[2074,2094,2109,2127,2149].codfw.wmnet with reason: Maintenance |
[production] |
| 03:44 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2105.codfw.wmnet with reason: Maintenance |
[production] |
| 03:44 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2105.codfw.wmnet with reason: Maintenance |
[production] |