|
2023-03-06
ยง
|
| 15:09 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1172.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:08 |
<Lucas_WMDE> |
UTC afternoon backport+config window done |
[production] |
| 15:06 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/termbox: apply |
[production] |
| 15:06 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [codfw] START helmfile.d/services/termbox: apply |
[production] |
| 15:05 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/termbox: apply |
[production] |
| 15:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2124 (T329260)', diff saved to https://phabricator.wikimedia.org/P45060 and previous config saved to /var/cache/conftool/dbconfig/20230306-150510-marostegui.json |
[production] |
| 15:04 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [eqiad] START helmfile.d/services/termbox: apply |
[production] |
| 15:02 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [staging] DONE helmfile.d/services/termbox: apply |
[production] |
| 15:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2127 (T328817)', diff saved to https://phabricator.wikimedia.org/P45059 and previous config saved to /var/cache/conftool/dbconfig/20230306-150115-marostegui.json |
[production] |
| 15:01 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2127.codfw.wmnet with reason: Maintenance |
[production] |
| 15:01 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2127.codfw.wmnet with reason: Maintenance |
[production] |
| 15:00 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2109 (T328817)', diff saved to https://phabricator.wikimedia.org/P45058 and previous config saved to /var/cache/conftool/dbconfig/20230306-150054-marostegui.json |
[production] |
| 14:59 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [staging] START helmfile.d/services/termbox: apply |
[production] |
| 14:59 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2124 (T329260)', diff saved to https://phabricator.wikimedia.org/P45057 and previous config saved to /var/cache/conftool/dbconfig/20230306-145945-marostegui.json |
[production] |
| 14:59 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2124.codfw.wmnet with reason: Maintenance |
[production] |
| 14:59 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2124.codfw.wmnet with reason: Maintenance |
[production] |
| 14:59 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2117 (T329260)', diff saved to https://phabricator.wikimedia.org/P45056 and previous config saved to /var/cache/conftool/dbconfig/20230306-145924-marostegui.json |
[production] |
| 14:57 |
<herron> |
failing grafana over to codfw T329073 |
[production] |
| 14:51 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1171.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:50 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1171.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:50 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1167 (T329203)', diff saved to https://phabricator.wikimedia.org/P45055 and previous config saved to /var/cache/conftool/dbconfig/20230306-145052-marostegui.json |
[production] |
| 14:50 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [staging] DONE helmfile.d/services/termbox: apply |
[production] |
| 14:49 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [staging] START helmfile.d/services/termbox: apply |
[production] |
| 14:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2109', diff saved to https://phabricator.wikimedia.org/P45054 and previous config saved to /var/cache/conftool/dbconfig/20230306-144547-marostegui.json |
[production] |
| 14:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2117', diff saved to https://phabricator.wikimedia.org/P45053 and previous config saved to /var/cache/conftool/dbconfig/20230306-144417-marostegui.json |
[production] |
| 14:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1167', diff saved to https://phabricator.wikimedia.org/P45051 and previous config saved to /var/cache/conftool/dbconfig/20230306-143546-marostegui.json |
[production] |
| 14:30 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2109', diff saved to https://phabricator.wikimedia.org/P45050 and previous config saved to /var/cache/conftool/dbconfig/20230306-143041-marostegui.json |
[production] |
| 14:29 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2117', diff saved to https://phabricator.wikimedia.org/P45049 and previous config saved to /var/cache/conftool/dbconfig/20230306-142910-marostegui.json |
[production] |
| 14:25 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [staging] DONE helmfile.d/services/termbox: apply |
[production] |
| 14:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1167', diff saved to https://phabricator.wikimedia.org/P45048 and previous config saved to /var/cache/conftool/dbconfig/20230306-142039-marostegui.json |
[production] |
| 14:16 |
<sukhe> |
running authdns-update for CR 894652 |
[production] |
| 14:15 |
<lucaswerkmeister-wmde@deploy2002> |
helmfile [staging] START helmfile.d/services/termbox: apply |
[production] |
| 14:15 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2109 (T328817)', diff saved to https://phabricator.wikimedia.org/P45047 and previous config saved to /var/cache/conftool/dbconfig/20230306-141534-marostegui.json |
[production] |
| 14:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2117 (T329260)', diff saved to https://phabricator.wikimedia.org/P45046 and previous config saved to /var/cache/conftool/dbconfig/20230306-141404-marostegui.json |
[production] |
| 14:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1167 (T329203)', diff saved to https://phabricator.wikimedia.org/P45045 and previous config saved to /var/cache/conftool/dbconfig/20230306-140533-marostegui.json |
[production] |
| 14:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2117 (T329260)', diff saved to https://phabricator.wikimedia.org/P45044 and previous config saved to /var/cache/conftool/dbconfig/20230306-140339-marostegui.json |
[production] |
| 14:03 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2117.codfw.wmnet with reason: Maintenance |
[production] |
| 14:03 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2117.codfw.wmnet with reason: Maintenance |
[production] |
| 14:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2114 (T329260)', diff saved to https://phabricator.wikimedia.org/P45043 and previous config saved to /var/cache/conftool/dbconfig/20230306-140317-marostegui.json |
[production] |
| 13:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2109 (T328817)', diff saved to https://phabricator.wikimedia.org/P45042 and previous config saved to /var/cache/conftool/dbconfig/20230306-134820-marostegui.json |
[production] |
| 13:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2114', diff saved to https://phabricator.wikimedia.org/P45041 and previous config saved to /var/cache/conftool/dbconfig/20230306-134811-marostegui.json |
[production] |
| 13:48 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2109.codfw.wmnet with reason: Maintenance |
[production] |
| 13:47 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2109.codfw.wmnet with reason: Maintenance |
[production] |
| 13:40 |
<filippo@cumin1001> |
conftool action : set/pooled=no; selector: name=thanos-fe1001.eqiad.wmnet,service=thanos-web |
[production] |
| 13:40 |
<filippo@cumin1001> |
conftool action : set/pooled=yes; selector: name=thanos-fe1002.eqiad.wmnet,service=thanos-web |
[production] |
| 13:35 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:34 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1198 (T328817)', diff saved to https://phabricator.wikimedia.org/P45040 and previous config saved to /var/cache/conftool/dbconfig/20230306-133451-marostegui.json |
[production] |
| 13:34 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.roll-restart-restbase (exit_code=0) rolling restart_daemons on A:restbase-canary |
[production] |
| 13:34 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.roll-restart-restbase rolling restart_daemons on A:restbase-canary |
[production] |