|
2022-11-28
ยง
|
| 13:45 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on ms-be2050.codfw.wmnet with reason: host reimage |
[production] |
| 13:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148', diff saved to https://phabricator.wikimedia.org/P41319 and previous config saved to /var/cache/conftool/dbconfig/20221128-133655-marostegui.json |
[production] |
| 13:36 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1122 (T323827)', diff saved to https://phabricator.wikimedia.org/P41318 and previous config saved to /var/cache/conftool/dbconfig/20221128-133648-ladsgroup.json |
[production] |
| 13:36 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 10:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:36 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 10:00:00 on db1122.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:36 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312 (T323827)', diff saved to https://phabricator.wikimedia.org/P41317 and previous config saved to /var/cache/conftool/dbconfig/20221128-133615-ladsgroup.json |
[production] |
| 13:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112', diff saved to https://phabricator.wikimedia.org/P41316 and previous config saved to /var/cache/conftool/dbconfig/20221128-133456-ladsgroup.json |
[production] |
| 13:32 |
<filippo@cumin1001> |
conftool action : set/pooled=false; selector: dnsdisc=thanos-query,name=eqiad |
[production] |
| 13:27 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.reimage for host ms-be2050.codfw.wmnet with OS bullseye |
[production] |
| 13:27 |
<jbond@cumin2002> |
END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ms-be2050.codfw.wmnet with OS bullseye |
[production] |
| 13:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2125 (T323827)', diff saved to https://phabricator.wikimedia.org/P41315 and previous config saved to /var/cache/conftool/dbconfig/20221128-132706-ladsgroup.json |
[production] |
| 13:27 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 10:00:00 on db2125.codfw.wmnet with reason: Maintenance |
[production] |
| 13:26 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 10:00:00 on db2125.codfw.wmnet with reason: Maintenance |
[production] |
| 13:26 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2104 (T323827)', diff saved to https://phabricator.wikimedia.org/P41314 and previous config saved to /var/cache/conftool/dbconfig/20221128-132645-ladsgroup.json |
[production] |
| 13:24 |
<godog> |
upgrade thanos on prometheus2* - T303154 |
[production] |
| 13:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2109 (T323827)', diff saved to https://phabricator.wikimedia.org/P41313 and previous config saved to /var/cache/conftool/dbconfig/20221128-132415-ladsgroup.json |
[production] |
| 13:24 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 10:00:00 on db2109.codfw.wmnet with reason: Maintenance |
[production] |
| 13:24 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 10:00:00 on db2109.codfw.wmnet with reason: Maintenance |
[production] |
| 13:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2105 (T323827)', diff saved to https://phabricator.wikimedia.org/P41312 and previous config saved to /var/cache/conftool/dbconfig/20221128-132404-ladsgroup.json |
[production] |
| 13:21 |
<godog> |
upgrade thanos on thanos-fe2* - T303154 |
[production] |
| 13:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148', diff saved to https://phabricator.wikimedia.org/P41311 and previous config saved to /var/cache/conftool/dbconfig/20221128-132149-marostegui.json |
[production] |
| 13:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312', diff saved to https://phabricator.wikimedia.org/P41310 and previous config saved to /var/cache/conftool/dbconfig/20221128-132109-ladsgroup.json |
[production] |
| 13:20 |
<moritzm> |
rebalance Ganeti group B/codfw following reboots |
[production] |
| 13:19 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112', diff saved to https://phabricator.wikimedia.org/P41309 and previous config saved to /var/cache/conftool/dbconfig/20221128-131949-ladsgroup.json |
[production] |
| 13:18 |
<godog> |
upgrade thanos on thanos-fe2001 - T303154 |
[production] |
| 13:16 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.reimage for host ms-be2050.codfw.wmnet with OS bullseye |
[production] |
| 13:11 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2104', diff saved to https://phabricator.wikimedia.org/P41308 and previous config saved to /var/cache/conftool/dbconfig/20221128-131138-ladsgroup.json |
[production] |
| 13:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2105', diff saved to https://phabricator.wikimedia.org/P41307 and previous config saved to /var/cache/conftool/dbconfig/20221128-130858-ladsgroup.json |
[production] |
| 13:06 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148 (T321126)', diff saved to https://phabricator.wikimedia.org/P41306 and previous config saved to /var/cache/conftool/dbconfig/20221128-130642-marostegui.json |
[production] |
| 13:06 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312', diff saved to https://phabricator.wikimedia.org/P41305 and previous config saved to /var/cache/conftool/dbconfig/20221128-130603-ladsgroup.json |
[production] |
| 13:04 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112 (T323907)', diff saved to https://phabricator.wikimedia.org/P41304 and previous config saved to /var/cache/conftool/dbconfig/20221128-130443-ladsgroup.json |
[production] |
| 12:56 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2104', diff saved to https://phabricator.wikimedia.org/P41303 and previous config saved to /var/cache/conftool/dbconfig/20221128-125632-ladsgroup.json |
[production] |
| 12:56 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1148.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:56 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1148.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:56 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1147 (T321126)', diff saved to https://phabricator.wikimedia.org/P41302 and previous config saved to /var/cache/conftool/dbconfig/20221128-125612-marostegui.json |
[production] |
| 12:53 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2105', diff saved to https://phabricator.wikimedia.org/P41301 and previous config saved to /var/cache/conftool/dbconfig/20221128-125351-ladsgroup.json |
[production] |
| 12:52 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1112 (T323907)', diff saved to https://phabricator.wikimedia.org/P41300 and previous config saved to /var/cache/conftool/dbconfig/20221128-125200-ladsgroup.json |
[production] |
| 12:51 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:51 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:51 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1112.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:51 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1112.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312 (T323827)', diff saved to https://phabricator.wikimedia.org/P41299 and previous config saved to /var/cache/conftool/dbconfig/20221128-125056-ladsgroup.json |
[production] |
| 12:47 |
<oblivian@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/termbox: apply |
[production] |
| 12:46 |
<oblivian@deploy1002> |
helmfile [eqiad] START helmfile.d/services/termbox: apply |
[production] |
| 12:45 |
<oblivian@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/termbox: apply |
[production] |
| 12:44 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/termbox: apply |
[production] |
| 12:44 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/termbox: apply |
[production] |
| 12:41 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2104 (T323827)', diff saved to https://phabricator.wikimedia.org/P41298 and previous config saved to /var/cache/conftool/dbconfig/20221128-124125-ladsgroup.json |
[production] |
| 12:41 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1147', diff saved to https://phabricator.wikimedia.org/P41297 and previous config saved to /var/cache/conftool/dbconfig/20221128-124105-marostegui.json |
[production] |
| 12:40 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/termbox: apply |
[production] |