|
2022-10-31
ยง
|
| 09:52 |
<gehel> |
depooling wdqs1007 while it catches up on lag - T322010 |
[production] |
| 09:47 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2108', diff saved to https://phabricator.wikimedia.org/P37079 and previous config saved to /var/cache/conftool/dbconfig/20221031-094748-ladsgroup.json |
[production] |
| 09:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1101:3317 (T318955)', diff saved to https://phabricator.wikimedia.org/P37078 and previous config saved to /var/cache/conftool/dbconfig/20221031-094501-ladsgroup.json |
[production] |
| 09:44 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1101.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:44 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1101.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:44 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3317 (T318955)', diff saved to https://phabricator.wikimedia.org/P37077 and previous config saved to /var/cache/conftool/dbconfig/20221031-094439-ladsgroup.json |
[production] |
| 09:40 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2104 (T318605)', diff saved to https://phabricator.wikimedia.org/P37076 and previous config saved to /var/cache/conftool/dbconfig/20221031-094045-ladsgroup.json |
[production] |
| 09:39 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3316', diff saved to https://phabricator.wikimedia.org/P37075 and previous config saved to /var/cache/conftool/dbconfig/20221031-093900-marostegui.json |
[production] |
| 09:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2108', diff saved to https://phabricator.wikimedia.org/P37074 and previous config saved to /var/cache/conftool/dbconfig/20221031-093242-ladsgroup.json |
[production] |
| 09:31 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2104 (T318605)', diff saved to https://phabricator.wikimedia.org/P37073 and previous config saved to /var/cache/conftool/dbconfig/20221031-093102-ladsgroup.json |
[production] |
| 09:30 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2104.codfw.wmnet with reason: Maintenance |
[production] |
| 09:30 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2104.codfw.wmnet with reason: Maintenance |
[production] |
| 09:30 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1105:3312 (T318605)', diff saved to https://phabricator.wikimedia.org/P37072 and previous config saved to /var/cache/conftool/dbconfig/20221031-093012-ladsgroup.json |
[production] |
| 09:30 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:29 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:29 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3317', diff saved to https://phabricator.wikimedia.org/P37071 and previous config saved to /var/cache/conftool/dbconfig/20221031-092933-ladsgroup.json |
[production] |
| 09:23 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3316 (T321123)', diff saved to https://phabricator.wikimedia.org/P37070 and previous config saved to /var/cache/conftool/dbconfig/20221031-092354-marostegui.json |
[production] |
| 09:22 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2169:3316 (T321123)', diff saved to https://phabricator.wikimedia.org/P37069 and previous config saved to /var/cache/conftool/dbconfig/20221031-092242-marostegui.json |
[production] |
| 09:22 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db2169.codfw.wmnet with reason: Maintenance |
[production] |
| 09:22 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db2169.codfw.wmnet with reason: Maintenance |
[production] |
| 09:22 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2158 (T321123)', diff saved to https://phabricator.wikimedia.org/P37068 and previous config saved to /var/cache/conftool/dbconfig/20221031-092221-marostegui.json |
[production] |
| 09:17 |
<Emperor> |
set thanos ring replicas to 3.40 T311690 |
[production] |
| 09:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2108 (T318955)', diff saved to https://phabricator.wikimedia.org/P37067 and previous config saved to /var/cache/conftool/dbconfig/20221031-091735-ladsgroup.json |
[production] |
| 09:14 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3317', diff saved to https://phabricator.wikimedia.org/P37066 and previous config saved to /var/cache/conftool/dbconfig/20221031-091426-ladsgroup.json |
[production] |
| 09:07 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2158', diff saved to https://phabricator.wikimedia.org/P37065 and previous config saved to /var/cache/conftool/dbconfig/20221031-090714-marostegui.json |
[production] |
| 09:06 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2108 (T318955)', diff saved to https://phabricator.wikimedia.org/P37064 and previous config saved to /var/cache/conftool/dbconfig/20221031-090640-ladsgroup.json |
[production] |
| 09:06 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db2108.codfw.wmnet with reason: Maintenance |
[production] |
| 09:06 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db2108.codfw.wmnet with reason: Maintenance |
[production] |
| 08:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3317 (T318955)', diff saved to https://phabricator.wikimedia.org/P37063 and previous config saved to /var/cache/conftool/dbconfig/20221031-085920-ladsgroup.json |
[production] |
| 08:58 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1166 (T318950)', diff saved to https://phabricator.wikimedia.org/P37062 and previous config saved to /var/cache/conftool/dbconfig/20221031-085839-ladsgroup.json |
[production] |
| 08:58 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1166.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:58 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1166.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:57 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db2100.codfw.wmnet with reason: Maintenance |
[production] |
| 08:57 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db2100.codfw.wmnet with reason: Maintenance |
[production] |
| 08:52 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2158', diff saved to https://phabricator.wikimedia.org/P37061 and previous config saved to /var/cache/conftool/dbconfig/20221031-085208-marostegui.json |
[production] |
| 08:50 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1100.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:50 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1100.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:48 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db2098.codfw.wmnet with reason: Maintenance |
[production] |
| 08:48 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db2098.codfw.wmnet with reason: Maintenance |
[production] |
| 08:47 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1098:3317 (T318955)', diff saved to https://phabricator.wikimedia.org/P37060 and previous config saved to /var/cache/conftool/dbconfig/20221031-084751-ladsgroup.json |
[production] |
| 08:47 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:47 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:47 |
<ladsgroup@cumin1001> |
END (ERROR) - Cookbook sre.hosts.downtime (exit_code=97) for 8:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:47 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:40 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:40 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2097.codfw.wmnet with reason: Maintenance |
[production] |
| 08:40 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:40 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2097.codfw.wmnet with reason: Maintenance |
[production] |
| 08:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2158 (T321123)', diff saved to https://phabricator.wikimedia.org/P37059 and previous config saved to /var/cache/conftool/dbconfig/20221031-083701-marostegui.json |
[production] |
| 08:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2158 (T321123)', diff saved to https://phabricator.wikimedia.org/P37058 and previous config saved to /var/cache/conftool/dbconfig/20221031-083449-marostegui.json |
[production] |