|
2023-08-22
ยง
|
| 06:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1212', diff saved to https://phabricator.wikimedia.org/P50792 and previous config saved to /var/cache/conftool/dbconfig/20230822-063258-ladsgroup.json |
[production] |
| 06:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1185', diff saved to https://phabricator.wikimedia.org/P50791 and previous config saved to /var/cache/conftool/dbconfig/20230822-063200-ladsgroup.json |
[production] |
| 06:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set db2112 with weight 0 T344666', diff saved to https://phabricator.wikimedia.org/P50790 and previous config saved to /var/cache/conftool/dbconfig/20230822-062854-ladsgroup.json |
[production] |
| 06:27 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 35 hosts with reason: Primary switchover s1 T344666 |
[production] |
| 06:26 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 35 hosts with reason: Primary switchover s1 T344666 |
[production] |
| 06:26 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2177', diff saved to https://phabricator.wikimedia.org/P50789 and previous config saved to /var/cache/conftool/dbconfig/20230822-062600-ladsgroup.json |
[production] |
| 06:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2137:3315', diff saved to https://phabricator.wikimedia.org/P50788 and previous config saved to /var/cache/conftool/dbconfig/20230822-062417-ladsgroup.json |
[production] |
| 06:21 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.reimage for host an-worker1115.eqiad.wmnet with OS bullseye |
[production] |
| 06:21 |
<stevemunene@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-worker1114.eqiad.wmnet with OS bullseye |
[production] |
| 06:19 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2112 (T344589)', diff saved to https://phabricator.wikimedia.org/P50787 and previous config saved to /var/cache/conftool/dbconfig/20230822-061956-ladsgroup.json |
[production] |
| 06:19 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2112.codfw.wmnet with reason: Maintenance |
[production] |
| 06:19 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2112.codfw.wmnet with reason: Maintenance |
[production] |
| 06:17 |
<stevemunene@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-worker1113.eqiad.wmnet with OS bullseye |
[production] |
| 06:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1212', diff saved to https://phabricator.wikimedia.org/P50786 and previous config saved to /var/cache/conftool/dbconfig/20230822-061752-ladsgroup.json |
[production] |
| 06:16 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1185', diff saved to https://phabricator.wikimedia.org/P50785 and previous config saved to /var/cache/conftool/dbconfig/20230822-061653-ladsgroup.json |
[production] |
| 06:15 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1163.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:15 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1163.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:10 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2177', diff saved to https://phabricator.wikimedia.org/P50784 and previous config saved to /var/cache/conftool/dbconfig/20230822-061054-ladsgroup.json |
[production] |
| 06:09 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1163.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:09 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1163.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2137:3315 (T343718)', diff saved to https://phabricator.wikimedia.org/P50783 and previous config saved to /var/cache/conftool/dbconfig/20230822-060911-ladsgroup.json |
[production] |
| 06:07 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depool db1163 T344621', diff saved to https://phabricator.wikimedia.org/P50782 and previous config saved to /var/cache/conftool/dbconfig/20230822-060710-ladsgroup.json |
[production] |
| 06:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1212 (T344589)', diff saved to https://phabricator.wikimedia.org/P50781 and previous config saved to /var/cache/conftool/dbconfig/20230822-060246-ladsgroup.json |
[production] |
| 06:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1185 (T343718)', diff saved to https://phabricator.wikimedia.org/P50780 and previous config saved to /var/cache/conftool/dbconfig/20230822-060147-ladsgroup.json |
[production] |
| 06:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Promote db1184 to s1 primary and set section read-write T344621', diff saved to https://phabricator.wikimedia.org/P50779 and previous config saved to /var/cache/conftool/dbconfig/20230822-060131-ladsgroup.json |
[production] |
| 06:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set s1 eqiad as read-only for maintenance - T344621', diff saved to https://phabricator.wikimedia.org/P50778 and previous config saved to /var/cache/conftool/dbconfig/20230822-060104-ladsgroup.json |
[production] |
| 06:00 |
<Amir1> |
Starting s1 eqiad failover from db1163 to db1184 - T344621 |
[production] |
| 05:58 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'configure' for AS: 12897 |
[production] |
| 05:57 |
<stevemunene@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-worker1114.eqiad.wmnet with reason: host reimage |
[production] |
| 05:56 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.peering with action 'configure' for AS: 12897 |
[production] |
| 05:55 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2177 (T344589)', diff saved to https://phabricator.wikimedia.org/P50777 and previous config saved to /var/cache/conftool/dbconfig/20230822-055548-ladsgroup.json |
[production] |
| 05:55 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1212 (T344589)', diff saved to https://phabricator.wikimedia.org/P50776 and previous config saved to /var/cache/conftool/dbconfig/20230822-055528-ladsgroup.json |
[production] |
| 05:55 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:55 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:55 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1212.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:55 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1212.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1198 (T344589)', diff saved to https://phabricator.wikimedia.org/P50775 and previous config saved to /var/cache/conftool/dbconfig/20230822-055457-ladsgroup.json |
[production] |
| 05:54 |
<stevemunene@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-worker1113.eqiad.wmnet with reason: host reimage |
[production] |
| 05:53 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on an-worker1114.eqiad.wmnet with reason: host reimage |
[production] |
| 05:51 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on an-worker1113.eqiad.wmnet with reason: host reimage |
[production] |
| 05:51 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2137:3315 (T343718)', diff saved to https://phabricator.wikimedia.org/P50774 and previous config saved to /var/cache/conftool/dbconfig/20230822-055101-ladsgroup.json |
[production] |
| 05:51 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2137.codfw.wmnet with reason: Maintenance |
[production] |
| 05:50 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2137.codfw.wmnet with reason: Maintenance |
[production] |
| 05:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2128 (T343718)', diff saved to https://phabricator.wikimedia.org/P50773 and previous config saved to /var/cache/conftool/dbconfig/20230822-055050-ladsgroup.json |
[production] |
| 05:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1185 (T343718)', diff saved to https://phabricator.wikimedia.org/P50772 and previous config saved to /var/cache/conftool/dbconfig/20230822-055040-ladsgroup.json |
[production] |
| 05:50 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1185.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:50 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1185.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161 (T343718)', diff saved to https://phabricator.wikimedia.org/P50771 and previous config saved to /var/cache/conftool/dbconfig/20230822-055030-ladsgroup.json |
[production] |
| 05:49 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2177 (T344589)', diff saved to https://phabricator.wikimedia.org/P50770 and previous config saved to /var/cache/conftool/dbconfig/20230822-054920-ladsgroup.json |
[production] |
| 05:49 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2177.codfw.wmnet with reason: Maintenance |
[production] |