|
2022-04-21
ยง
|
| 21:04 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 21:04 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1157 (T298565)', diff saved to https://phabricator.wikimedia.org/P26060 and previous config saved to /var/cache/conftool/dbconfig/20220421-210414-ladsgroup.json |
[production] |
| 21:04 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1157.eqiad.wmnet with reason: Maintenance |
[production] |
| 21:04 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1157.eqiad.wmnet with reason: Maintenance |
[production] |
| 20:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136 (T298565)', diff saved to https://phabricator.wikimedia.org/P26059 and previous config saved to /var/cache/conftool/dbconfig/20220421-205906-ladsgroup.json |
[production] |
| 20:52 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182 (T298565)', diff saved to https://phabricator.wikimedia.org/P26058 and previous config saved to /var/cache/conftool/dbconfig/20220421-205256-ladsgroup.json |
[production] |
| 20:51 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1131', diff saved to https://phabricator.wikimedia.org/P26057 and previous config saved to /var/cache/conftool/dbconfig/20220421-205145-ladsgroup.json |
[production] |
| 20:50 |
<cdanis> |
re-enabled puppet and repooled cp2029 |
[production] |
| 20:47 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1157 (T298565)', diff saved to https://phabricator.wikimedia.org/P26056 and previous config saved to /var/cache/conftool/dbconfig/20220421-204709-ladsgroup.json |
[production] |
| 20:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1136 (T298565)', diff saved to https://phabricator.wikimedia.org/P26055 and previous config saved to /var/cache/conftool/dbconfig/20220421-204532-ladsgroup.json |
[production] |
| 20:45 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 20:45 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 20:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136 (T298565)', diff saved to https://phabricator.wikimedia.org/P26054 and previous config saved to /var/cache/conftool/dbconfig/20220421-204508-ladsgroup.json |
[production] |
| 20:45 |
<dzahn@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on gitlab-runner2001.codfw.wmnet with reason: reimage |
[production] |
| 20:45 |
<dzahn@cumin2002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on gitlab-runner2001.codfw.wmnet with reason: reimage |
[production] |
| 20:41 |
<nokafor@deploy1002> |
Finished deploy [airflow-dags/analytics@bd28d80]: (no justification provided) (duration: 00m 07s) |
[production] |
| 20:41 |
<nokafor@deploy1002> |
Started deploy [airflow-dags/analytics@bd28d80]: (no justification provided) |
[production] |
| 20:39 |
<nokafor@deploy1002> |
Finished deploy [airflow-dags/analytics@bd28d80]: (no justification provided) (duration: 00m 27s) |
[production] |
| 20:39 |
<nokafor@deploy1002> |
Started deploy [airflow-dags/analytics@bd28d80]: (no justification provided) |
[production] |
| 20:36 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1131', diff saved to https://phabricator.wikimedia.org/P26053 and previous config saved to /var/cache/conftool/dbconfig/20220421-203640-ladsgroup.json |
[production] |
| 20:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1157', diff saved to https://phabricator.wikimedia.org/P26052 and previous config saved to /var/cache/conftool/dbconfig/20220421-203204-ladsgroup.json |
[production] |
| 20:30 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136', diff saved to https://phabricator.wikimedia.org/P26051 and previous config saved to /var/cache/conftool/dbconfig/20220421-203003-ladsgroup.json |
[production] |
| 20:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1182 (T298565)', diff saved to https://phabricator.wikimedia.org/P26050 and previous config saved to /var/cache/conftool/dbconfig/20220421-202826-ladsgroup.json |
[production] |
| 20:28 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1182.eqiad.wmnet with reason: Maintenance |
[production] |
| 20:28 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1182.eqiad.wmnet with reason: Maintenance |
[production] |
| 20:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1156 (T298565)', diff saved to https://phabricator.wikimedia.org/P26049 and previous config saved to /var/cache/conftool/dbconfig/20220421-202818-ladsgroup.json |
[production] |
| 20:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1131 (T306560)', diff saved to https://phabricator.wikimedia.org/P26048 and previous config saved to /var/cache/conftool/dbconfig/20220421-202135-ladsgroup.json |
[production] |
| 20:18 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1131 (T306560)', diff saved to https://phabricator.wikimedia.org/P26047 and previous config saved to /var/cache/conftool/dbconfig/20220421-201825-ladsgroup.json |
[production] |
| 20:18 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1131.eqiad.wmnet with reason: Maintenance |
[production] |
| 20:18 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1131.eqiad.wmnet with reason: Maintenance |
[production] |
| 20:18 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1165 (T306560)', diff saved to https://phabricator.wikimedia.org/P26046 and previous config saved to /var/cache/conftool/dbconfig/20220421-201817-ladsgroup.json |
[production] |
| 20:16 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1157', diff saved to https://phabricator.wikimedia.org/P26045 and previous config saved to /var/cache/conftool/dbconfig/20220421-201659-ladsgroup.json |
[production] |
| 20:14 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136', diff saved to https://phabricator.wikimedia.org/P26044 and previous config saved to /var/cache/conftool/dbconfig/20220421-201455-ladsgroup.json |
[production] |
| 20:14 |
<mutante> |
reimaging gitlab-runner2001.codfw.wmnet one more time to confirm things work from scratch now T297659 |
[production] |
| 20:13 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1156', diff saved to https://phabricator.wikimedia.org/P26043 and previous config saved to /var/cache/conftool/dbconfig/20220421-201313-ladsgroup.json |
[production] |
| 20:09 |
<mutante> |
[ganeti2021:~] $ sudo gnt-instance shutdown gitlab-runner2001.codfw.wmnet |
[production] |
| 20:08 |
<mutante> |
[puppetmaster1001:~] $ sudo puppet cert clean gitlab-runner2001.codfw.wmnet |
[production] |
| 20:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1165', diff saved to https://phabricator.wikimedia.org/P26042 and previous config saved to /var/cache/conftool/dbconfig/20220421-200312-ladsgroup.json |
[production] |
| 20:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1157 (T298565)', diff saved to https://phabricator.wikimedia.org/P26041 and previous config saved to /var/cache/conftool/dbconfig/20220421-200154-ladsgroup.json |
[production] |
| 19:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1136 (T298565)', diff saved to https://phabricator.wikimedia.org/P26040 and previous config saved to /var/cache/conftool/dbconfig/20220421-195950-ladsgroup.json |
[production] |
| 19:58 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1156', diff saved to https://phabricator.wikimedia.org/P26039 and previous config saved to /var/cache/conftool/dbconfig/20220421-195808-ladsgroup.json |
[production] |
| 19:48 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1165', diff saved to https://phabricator.wikimedia.org/P26038 and previous config saved to /var/cache/conftool/dbconfig/20220421-194807-ladsgroup.json |
[production] |
| 19:44 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1136 (T298565)', diff saved to https://phabricator.wikimedia.org/P26037 and previous config saved to /var/cache/conftool/dbconfig/20220421-194419-ladsgroup.json |
[production] |
| 19:44 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 19:44 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 19:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1156 (T298565)', diff saved to https://phabricator.wikimedia.org/P26036 and previous config saved to /var/cache/conftool/dbconfig/20220421-194303-ladsgroup.json |
[production] |
| 19:33 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1165 (T306560)', diff saved to https://phabricator.wikimedia.org/P26035 and previous config saved to /var/cache/conftool/dbconfig/20220421-193302-ladsgroup.json |
[production] |
| 19:31 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1165 (T306560)', diff saved to https://phabricator.wikimedia.org/P26034 and previous config saved to /var/cache/conftool/dbconfig/20220421-193052-ladsgroup.json |
[production] |
| 19:30 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on clouddb[1015,1019,1021].eqiad.wmnet,db1155.eqiad.wmnet with reason: Maintenance |
[production] |
| 19:30 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on clouddb[1015,1019,1021].eqiad.wmnet,db1155.eqiad.wmnet with reason: Maintenance |
[production] |