|
2022-11-29
ยง
|
| 09:11 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1107.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:11 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1107.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:11 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1106 (T321126)', diff saved to https://phabricator.wikimedia.org/P41635 and previous config saved to /var/cache/conftool/dbconfig/20221129-091117-marostegui.json |
[production] |
| 09:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2108 (T323907)', diff saved to https://phabricator.wikimedia.org/P41634 and previous config saved to /var/cache/conftool/dbconfig/20221129-090237-ladsgroup.json |
[production] |
| 09:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1191 (T323907)', diff saved to https://phabricator.wikimedia.org/P41633 and previous config saved to /var/cache/conftool/dbconfig/20221129-090044-ladsgroup.json |
[production] |
| 09:00 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1191.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:00 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1191.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1181 (T323907)', diff saved to https://phabricator.wikimedia.org/P41632 and previous config saved to /var/cache/conftool/dbconfig/20221129-090023-ladsgroup.json |
[production] |
| 08:58 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2109', diff saved to https://phabricator.wikimedia.org/P41631 and previous config saved to /var/cache/conftool/dbconfig/20221129-085809-ladsgroup.json |
[production] |
| 08:56 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1106', diff saved to https://phabricator.wikimedia.org/P41630 and previous config saved to /var/cache/conftool/dbconfig/20221129-085611-marostegui.json |
[production] |
| 08:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1181', diff saved to https://phabricator.wikimedia.org/P41629 and previous config saved to /var/cache/conftool/dbconfig/20221129-084517-ladsgroup.json |
[production] |
| 08:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2109 (T322618)', diff saved to https://phabricator.wikimedia.org/P41628 and previous config saved to /var/cache/conftool/dbconfig/20221129-084302-ladsgroup.json |
[production] |
| 08:41 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1106', diff saved to https://phabricator.wikimedia.org/P41627 and previous config saved to /var/cache/conftool/dbconfig/20221129-084104-marostegui.json |
[production] |
| 08:35 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2109 (T322618)', diff saved to https://phabricator.wikimedia.org/P41626 and previous config saved to /var/cache/conftool/dbconfig/20221129-083521-ladsgroup.json |
[production] |
| 08:35 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2109.codfw.wmnet with reason: Maintenance |
[production] |
| 08:35 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2109.codfw.wmnet with reason: Maintenance |
[production] |
| 08:35 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2105 (T322618)', diff saved to https://phabricator.wikimedia.org/P41625 and previous config saved to /var/cache/conftool/dbconfig/20221129-083511-ladsgroup.json |
[production] |
| 08:30 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1181', diff saved to https://phabricator.wikimedia.org/P41624 and previous config saved to /var/cache/conftool/dbconfig/20221129-083010-ladsgroup.json |
[production] |
| 08:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2108 (T323907)', diff saved to https://phabricator.wikimedia.org/P41623 and previous config saved to /var/cache/conftool/dbconfig/20221129-082740-ladsgroup.json |
[production] |
| 08:27 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2108.codfw.wmnet with reason: Maintenance |
[production] |
| 08:27 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2108.codfw.wmnet with reason: Maintenance |
[production] |
| 08:25 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1106 (T321126)', diff saved to https://phabricator.wikimedia.org/P41622 and previous config saved to /var/cache/conftool/dbconfig/20221129-082558-marostegui.json |
[production] |
| 08:24 |
<oblivian@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=1) for host mw1457.eqiad.wmnet |
[production] |
| 08:23 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1106 (T321126)', diff saved to https://phabricator.wikimedia.org/P41621 and previous config saved to /var/cache/conftool/dbconfig/20221129-082335-marostegui.json |
[production] |
| 08:23 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 10:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:23 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 10:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:23 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1106.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:23 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1106.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:23 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3311 (T321126)', diff saved to https://phabricator.wikimedia.org/P41620 and previous config saved to /var/cache/conftool/dbconfig/20221129-082307-marostegui.json |
[production] |
| 08:20 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2105', diff saved to https://phabricator.wikimedia.org/P41619 and previous config saved to /var/cache/conftool/dbconfig/20221129-082004-ladsgroup.json |
[production] |
| 08:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1181 (T323907)', diff saved to https://phabricator.wikimedia.org/P41618 and previous config saved to /var/cache/conftool/dbconfig/20221129-081504-ladsgroup.json |
[production] |
| 08:13 |
<oblivian@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host mw1457.eqiad.wmnet |
[production] |
| 08:13 |
<moritzm> |
rebalance Ganeti group D/codfw following reboots |
[production] |
| 08:08 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3311', diff saved to https://phabricator.wikimedia.org/P41614 and previous config saved to /var/cache/conftool/dbconfig/20221129-080801-marostegui.json |
[production] |
| 08:04 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2105', diff saved to https://phabricator.wikimedia.org/P41613 and previous config saved to /var/cache/conftool/dbconfig/20221129-080458-ladsgroup.json |
[production] |
| 08:03 |
<oblivian@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3:00:00 on 42 hosts with reason: Appservers |
[production] |
| 08:00 |
<oblivian@cumin1001> |
START - Cookbook sre.hosts.downtime for 3:00:00 on 42 hosts with reason: Appservers |
[production] |
| 07:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1181 (T323907)', diff saved to https://phabricator.wikimedia.org/P41612 and previous config saved to /var/cache/conftool/dbconfig/20221129-075937-ladsgroup.json |
[production] |
| 07:59 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:59 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:58 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1174 (T323907)', diff saved to https://phabricator.wikimedia.org/P41611 and previous config saved to /var/cache/conftool/dbconfig/20221129-075854-ladsgroup.json |
[production] |
| 07:55 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2100.codfw.wmnet with reason: Maintenance |
[production] |
| 07:55 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2100.codfw.wmnet with reason: Maintenance |
[production] |
| 07:52 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3311', diff saved to https://phabricator.wikimedia.org/P41610 and previous config saved to /var/cache/conftool/dbconfig/20221129-075254-marostegui.json |
[production] |
| 07:49 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2105 (T322618)', diff saved to https://phabricator.wikimedia.org/P41609 and previous config saved to /var/cache/conftool/dbconfig/20221129-074951-ladsgroup.json |
[production] |
| 07:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2174 (re)pooling @ 100%: After HW maintenance', diff saved to https://phabricator.wikimedia.org/P41608 and previous config saved to /var/cache/conftool/dbconfig/20221129-074441-root.json |
[production] |
| 07:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1174', diff saved to https://phabricator.wikimedia.org/P41607 and previous config saved to /var/cache/conftool/dbconfig/20221129-074347-ladsgroup.json |
[production] |
| 07:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2105 (T322618)', diff saved to https://phabricator.wikimedia.org/P41606 and previous config saved to /var/cache/conftool/dbconfig/20221129-074229-ladsgroup.json |
[production] |
| 07:42 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2105.codfw.wmnet with reason: Maintenance |
[production] |
| 07:42 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2105.codfw.wmnet with reason: Maintenance |
[production] |