|
2022-04-21
ยง
|
| 22:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136', diff saved to https://phabricator.wikimedia.org/P26091 and previous config saved to /var/cache/conftool/dbconfig/20220421-223233-ladsgroup.json |
[production] |
| 22:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3316', diff saved to https://phabricator.wikimedia.org/P26090 and previous config saved to /var/cache/conftool/dbconfig/20220421-222817-ladsgroup.json |
[production] |
| 22:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1162 (T298565)', diff saved to https://phabricator.wikimedia.org/P26089 and previous config saved to /var/cache/conftool/dbconfig/20220421-222550-ladsgroup.json |
[production] |
| 22:25 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1162.eqiad.wmnet with reason: Maintenance |
[production] |
| 22:25 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1162.eqiad.wmnet with reason: Maintenance |
[production] |
| 22:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1129 (T298565)', diff saved to https://phabricator.wikimedia.org/P26088 and previous config saved to /var/cache/conftool/dbconfig/20220421-222542-ladsgroup.json |
[production] |
| 22:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112 (T298565)', diff saved to https://phabricator.wikimedia.org/P26087 and previous config saved to /var/cache/conftool/dbconfig/20220421-222534-ladsgroup.json |
[production] |
| 22:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136 (T298565)', diff saved to https://phabricator.wikimedia.org/P26086 and previous config saved to /var/cache/conftool/dbconfig/20220421-221728-ladsgroup.json |
[production] |
| 22:13 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3316', diff saved to https://phabricator.wikimedia.org/P26085 and previous config saved to /var/cache/conftool/dbconfig/20220421-221312-ladsgroup.json |
[production] |
| 22:10 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1129', diff saved to https://phabricator.wikimedia.org/P26084 and previous config saved to /var/cache/conftool/dbconfig/20220421-221037-ladsgroup.json |
[production] |
| 22:05 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1112 (T298565)', diff saved to https://phabricator.wikimedia.org/P26083 and previous config saved to /var/cache/conftool/dbconfig/20220421-220552-ladsgroup.json |
[production] |
| 22:05 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 22:05 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 22:05 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1112.eqiad.wmnet with reason: Maintenance |
[production] |
| 22:05 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1112.eqiad.wmnet with reason: Maintenance |
[production] |
| 22:05 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1157 (T298565)', diff saved to https://phabricator.wikimedia.org/P26082 and previous config saved to /var/cache/conftool/dbconfig/20220421-220539-ladsgroup.json |
[production] |
| 22:02 |
<mutante> |
gitlab-runner2001 - systemctl start docker-resource-monitor ; systemctl start docker-gc |
[production] |
| 22:00 |
<mutante> |
gitlab-runner2001 - installing apparmor ('apparmor' is the user utilities package and was NOT installed, libapparmor1 WAS installed), this caused bug https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1808456.html after upgrading gitlab-runner to bullseye because bullseye comes with libapparmor1 by default as opposed to before T297659 |
[production] |
| 21:58 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3316 (T306560)', diff saved to https://phabricator.wikimedia.org/P26081 and previous config saved to /var/cache/conftool/dbconfig/20220421-215807-ladsgroup.json |
[production] |
| 21:55 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1096:3316 (T306560)', diff saved to https://phabricator.wikimedia.org/P26080 and previous config saved to /var/cache/conftool/dbconfig/20220421-215547-ladsgroup.json |
[production] |
| 21:55 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1096.eqiad.wmnet with reason: Maintenance |
[production] |
| 21:55 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1096.eqiad.wmnet with reason: Maintenance |
[production] |
| 21:55 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3316 (T306560)', diff saved to https://phabricator.wikimedia.org/P26079 and previous config saved to /var/cache/conftool/dbconfig/20220421-215540-ladsgroup.json |
[production] |
| 21:55 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1129', diff saved to https://phabricator.wikimedia.org/P26078 and previous config saved to /var/cache/conftool/dbconfig/20220421-215532-ladsgroup.json |
[production] |
| 21:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1157', diff saved to https://phabricator.wikimedia.org/P26077 and previous config saved to /var/cache/conftool/dbconfig/20220421-215034-ladsgroup.json |
[production] |
| 21:44 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1136 (T298565)', diff saved to https://phabricator.wikimedia.org/P26076 and previous config saved to /var/cache/conftool/dbconfig/20220421-214445-ladsgroup.json |
[production] |
| 21:44 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 21:44 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 21:44 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136 (T298565)', diff saved to https://phabricator.wikimedia.org/P26075 and previous config saved to /var/cache/conftool/dbconfig/20220421-214422-ladsgroup.json |
[production] |
| 21:42 |
<mutante> |
shutting down and reimaging gitlab-runner1001 T297659 |
[production] |
| 21:40 |
<dzahn@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on gitlab-runner1001.eqiad.wmnet with reason: reimage |
[production] |
| 21:40 |
<dzahn@cumin2002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on gitlab-runner1001.eqiad.wmnet with reason: reimage |
[production] |
| 21:40 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3316', diff saved to https://phabricator.wikimedia.org/P26074 and previous config saved to /var/cache/conftool/dbconfig/20220421-214035-ladsgroup.json |
[production] |
| 21:40 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1129 (T298565)', diff saved to https://phabricator.wikimedia.org/P26073 and previous config saved to /var/cache/conftool/dbconfig/20220421-214027-ladsgroup.json |
[production] |
| 21:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1129 (T298565)', diff saved to https://phabricator.wikimedia.org/P26072 and previous config saved to /var/cache/conftool/dbconfig/20220421-213819-ladsgroup.json |
[production] |
| 21:38 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1129.eqiad.wmnet with reason: Maintenance |
[production] |
| 21:38 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1129.eqiad.wmnet with reason: Maintenance |
[production] |
| 21:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182 (T298565)', diff saved to https://phabricator.wikimedia.org/P26071 and previous config saved to /var/cache/conftool/dbconfig/20220421-213811-ladsgroup.json |
[production] |
| 21:35 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1157', diff saved to https://phabricator.wikimedia.org/P26070 and previous config saved to /var/cache/conftool/dbconfig/20220421-213529-ladsgroup.json |
[production] |
| 21:29 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136', diff saved to https://phabricator.wikimedia.org/P26069 and previous config saved to /var/cache/conftool/dbconfig/20220421-212916-ladsgroup.json |
[production] |
| 21:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3316', diff saved to https://phabricator.wikimedia.org/P26068 and previous config saved to /var/cache/conftool/dbconfig/20220421-212523-ladsgroup.json |
[production] |
| 21:23 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182', diff saved to https://phabricator.wikimedia.org/P26067 and previous config saved to /var/cache/conftool/dbconfig/20220421-212306-ladsgroup.json |
[production] |
| 21:20 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1157 (T298565)', diff saved to https://phabricator.wikimedia.org/P26066 and previous config saved to /var/cache/conftool/dbconfig/20220421-212022-ladsgroup.json |
[production] |
| 21:19 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 21:19 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 21:19 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 21:19 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 21:14 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136', diff saved to https://phabricator.wikimedia.org/P26065 and previous config saved to /var/cache/conftool/dbconfig/20220421-211411-ladsgroup.json |
[production] |
| 21:10 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3316 (T306560)', diff saved to https://phabricator.wikimedia.org/P26064 and previous config saved to /var/cache/conftool/dbconfig/20220421-211018-ladsgroup.json |
[production] |
| 21:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182', diff saved to https://phabricator.wikimedia.org/P26063 and previous config saved to /var/cache/conftool/dbconfig/20220421-210801-ladsgroup.json |
[production] |