|
2024-05-02
ยง
|
| 06:57 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2147 (T361627)', diff saved to https://phabricator.wikimedia.org/P61668 and previous config saved to /var/cache/conftool/dbconfig/20240502-065758-marostegui.json |
[production] |
| 06:45 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2147 (T361627)', diff saved to https://phabricator.wikimedia.org/P61667 and previous config saved to /var/cache/conftool/dbconfig/20240502-064533-marostegui.json |
[production] |
| 06:45 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2147.codfw.wmnet with reason: Maintenance |
[production] |
| 06:45 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on db2147.codfw.wmnet with reason: Maintenance |
[production] |
| 06:42 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db2162 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P61666 and previous config saved to /var/cache/conftool/dbconfig/20240502-064230-root.json |
[production] |
| 06:33 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2139.codfw.wmnet with reason: Maintenance |
[production] |
| 06:33 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on db2139.codfw.wmnet with reason: Maintenance |
[production] |
| 06:33 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2137 (T361627)', diff saved to https://phabricator.wikimedia.org/P61665 and previous config saved to /var/cache/conftool/dbconfig/20240502-063343-marostegui.json |
[production] |
| 06:27 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db2162 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P61664 and previous config saved to /var/cache/conftool/dbconfig/20240502-062725-root.json |
[production] |
| 06:18 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2137', diff saved to https://phabricator.wikimedia.org/P61663 and previous config saved to /var/cache/conftool/dbconfig/20240502-061836-marostegui.json |
[production] |
| 06:12 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db2162 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P61662 and previous config saved to /var/cache/conftool/dbconfig/20240502-061218-root.json |
[production] |
| 06:03 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2137', diff saved to https://phabricator.wikimedia.org/P61661 and previous config saved to /var/cache/conftool/dbconfig/20240502-060328-marostegui.json |
[production] |
| 05:57 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db2162 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P61660 and previous config saved to /var/cache/conftool/dbconfig/20240502-055712-root.json |
[production] |
| 05:48 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2137 (T361627)', diff saved to https://phabricator.wikimedia.org/P61659 and previous config saved to /var/cache/conftool/dbconfig/20240502-054821-marostegui.json |
[production] |
| 05:48 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:48 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:42 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db2162 (re)pooling @ 10%: Repooling', diff saved to https://phabricator.wikimedia.org/P61658 and previous config saved to /var/cache/conftool/dbconfig/20240502-054206-root.json |
[production] |
| 05:37 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2137 (T361627)', diff saved to https://phabricator.wikimedia.org/P61657 and previous config saved to /var/cache/conftool/dbconfig/20240502-053717-marostegui.json |
[production] |
| 05:37 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2137.codfw.wmnet with reason: Maintenance |
[production] |
| 05:36 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on db2137.codfw.wmnet with reason: Maintenance |
[production] |
| 05:36 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2136 (T361627)', diff saved to https://phabricator.wikimedia.org/P61656 and previous config saved to /var/cache/conftool/dbconfig/20240502-053654-marostegui.json |
[production] |
| 05:27 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db1181.eqiad.wmnet with OS bookworm |
[production] |
| 05:27 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db2162 (re)pooling @ 5%: Repooling', diff saved to https://phabricator.wikimedia.org/P61655 and previous config saved to /var/cache/conftool/dbconfig/20240502-052700-root.json |
[production] |
| 05:21 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2136', diff saved to https://phabricator.wikimedia.org/P61654 and previous config saved to /var/cache/conftool/dbconfig/20240502-052146-marostegui.json |
[production] |
| 05:21 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:21 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:17 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db2162.codfw.wmnet with OS bookworm |
[production] |
| 05:11 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db2162 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P61653 and previous config saved to /var/cache/conftool/dbconfig/20240502-051155-root.json |
[production] |
| 05:06 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db1181.eqiad.wmnet with reason: host reimage |
[production] |
| 05:06 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2136', diff saved to https://phabricator.wikimedia.org/P61652 and previous config saved to /var/cache/conftool/dbconfig/20240502-050639-marostegui.json |
[production] |
| 05:04 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db1181.eqiad.wmnet with reason: host reimage |
[production] |
| 04:55 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db2162.codfw.wmnet with reason: host reimage |
[production] |
| 04:53 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2162.codfw.wmnet with reason: host reimage |
[production] |
| 04:52 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.reimage for host db1181.eqiad.wmnet with OS bookworm |
[production] |
| 04:51 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2136 (T361627)', diff saved to https://phabricator.wikimedia.org/P61651 and previous config saved to /var/cache/conftool/dbconfig/20240502-045131-marostegui.json |
[production] |
| 04:50 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depool db1181 T363892', diff saved to https://phabricator.wikimedia.org/P61650 and previous config saved to /var/cache/conftool/dbconfig/20240502-045017-root.json |
[production] |
| 04:48 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Promote db1236 to s7 primary and set section read-write T363892', diff saved to https://phabricator.wikimedia.org/P61649 and previous config saved to /var/cache/conftool/dbconfig/20240502-044848-marostegui.json |
[production] |
| 04:48 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Set s7 eqiad as read-only for maintenance - T363892', diff saved to https://phabricator.wikimedia.org/P61648 and previous config saved to /var/cache/conftool/dbconfig/20240502-044819-marostegui.json |
[production] |
| 04:48 |
<marostegui> |
Starting s7 eqiad failover from db1181 to db1236 - T363892 |
[production] |
| 04:40 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2136 (T361627)', diff saved to https://phabricator.wikimedia.org/P61647 and previous config saved to /var/cache/conftool/dbconfig/20240502-044020-marostegui.json |
[production] |
| 04:40 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2136.codfw.wmnet with reason: Maintenance |
[production] |
| 04:39 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on db2136.codfw.wmnet with reason: Maintenance |
[production] |
| 04:35 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.reimage for host db2162.codfw.wmnet with OS bookworm |
[production] |
| 04:34 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db2162.codfw.wmnet with reason: Reimage |
[production] |
| 04:34 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2162.codfw.wmnet with reason: Reimage |
[production] |
| 04:34 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depool db2162', diff saved to https://phabricator.wikimedia.org/P61646 and previous config saved to /var/cache/conftool/dbconfig/20240502-043403-root.json |
[production] |
| 04:30 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 28 hosts with reason: Primary switchover s7 T363892 |
[production] |
| 04:30 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Set db1236 with weight 0 T363892', diff saved to https://phabricator.wikimedia.org/P61645 and previous config saved to /var/cache/conftool/dbconfig/20240502-043019-marostegui.json |
[production] |
| 04:30 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 28 hosts with reason: Primary switchover s7 T363892 |
[production] |
| 04:29 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db2099.codfw.wmnet with reason: Maintenance |
[production] |