|
2022-06-28
ยง
|
| 11:44 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 11:44 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 11:43 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 11:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T298557)', diff saved to https://phabricator.wikimedia.org/P30574 and previous config saved to /var/cache/conftool/dbconfig/20220628-114020-marostegui.json |
[production] |
| 11:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1168 (T298557)', diff saved to https://phabricator.wikimedia.org/P30573 and previous config saved to /var/cache/conftool/dbconfig/20220628-113154-marostegui.json |
[production] |
| 11:31 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1168.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:31 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1168.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3316 (T298557)', diff saved to https://phabricator.wikimedia.org/P30572 and previous config saved to /var/cache/conftool/dbconfig/20220628-113146-marostegui.json |
[production] |
| 11:19 |
<moritzm> |
installing squid security updates |
[production] |
| 11:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3316', diff saved to https://phabricator.wikimedia.org/P30571 and previous config saved to /var/cache/conftool/dbconfig/20220628-111641-marostegui.json |
[production] |
| 11:08 |
<btullis@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host stat1010.eqiad.wmnet with OS bullseye |
[production] |
| 11:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3316', diff saved to https://phabricator.wikimedia.org/P30570 and previous config saved to /var/cache/conftool/dbconfig/20220628-110136-marostegui.json |
[production] |
| 10:55 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1141 (T309311)', diff saved to https://phabricator.wikimedia.org/P30569 and previous config saved to /var/cache/conftool/dbconfig/20220628-105551-ladsgroup.json |
[production] |
| 10:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3316 (T298557)', diff saved to https://phabricator.wikimedia.org/P30568 and previous config saved to /var/cache/conftool/dbconfig/20220628-104631-marostegui.json |
[production] |
| 10:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1161 (re)pooling @ 100%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30567 and previous config saved to /var/cache/conftool/dbconfig/20220628-104337-root.json |
[production] |
| 10:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 100%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30566 and previous config saved to /var/cache/conftool/dbconfig/20220628-104252-root.json |
[production] |
| 10:40 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1141', diff saved to https://phabricator.wikimedia.org/P30565 and previous config saved to /var/cache/conftool/dbconfig/20220628-104046-ladsgroup.json |
[production] |
| 10:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1161 (re)pooling @ 75%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30564 and previous config saved to /var/cache/conftool/dbconfig/20220628-102833-root.json |
[production] |
| 10:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 75%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30563 and previous config saved to /var/cache/conftool/dbconfig/20220628-102748-root.json |
[production] |
| 10:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1141', diff saved to https://phabricator.wikimedia.org/P30562 and previous config saved to /var/cache/conftool/dbconfig/20220628-102540-ladsgroup.json |
[production] |
| 10:23 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1096:3316 (T298557)', diff saved to https://phabricator.wikimedia.org/P30561 and previous config saved to /var/cache/conftool/dbconfig/20220628-102331-marostegui.json |
[production] |
| 10:23 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1096.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:23 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1096.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:23 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3316 (T298557)', diff saved to https://phabricator.wikimedia.org/P30560 and previous config saved to /var/cache/conftool/dbconfig/20220628-102322-marostegui.json |
[production] |
| 10:13 |
<moritzm> |
upgrading Ganeti test cluster to 3.0.2 |
[production] |
| 10:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1161 (re)pooling @ 50%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30559 and previous config saved to /var/cache/conftool/dbconfig/20220628-101329-root.json |
[production] |
| 10:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 50%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30558 and previous config saved to /var/cache/conftool/dbconfig/20220628-101244-root.json |
[production] |
| 10:11 |
<btullis@cumin1001> |
START - Cookbook sre.hosts.reimage for host stat1010.eqiad.wmnet with OS bullseye |
[production] |
| 10:10 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1141 (T309311)', diff saved to https://phabricator.wikimedia.org/P30557 and previous config saved to /var/cache/conftool/dbconfig/20220628-101035-ladsgroup.json |
[production] |
| 10:08 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3316', diff saved to https://phabricator.wikimedia.org/P30556 and previous config saved to /var/cache/conftool/dbconfig/20220628-100817-marostegui.json |
[production] |
| 09:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1141 (T309311)', diff saved to https://phabricator.wikimedia.org/P30555 and previous config saved to /var/cache/conftool/dbconfig/20220628-095927-ladsgroup.json |
[production] |
| 09:59 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1141.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:59 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1141.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1138 (T309311)', diff saved to https://phabricator.wikimedia.org/P30554 and previous config saved to /var/cache/conftool/dbconfig/20220628-095919-ladsgroup.json |
[production] |
| 09:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1161 (re)pooling @ 25%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30553 and previous config saved to /var/cache/conftool/dbconfig/20220628-095825-root.json |
[production] |
| 09:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 25%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30552 and previous config saved to /var/cache/conftool/dbconfig/20220628-095741-root.json |
[production] |
| 09:53 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3316', diff saved to https://phabricator.wikimedia.org/P30551 and previous config saved to /var/cache/conftool/dbconfig/20220628-095312-marostegui.json |
[production] |
| 09:44 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1138', diff saved to https://phabricator.wikimedia.org/P30550 and previous config saved to /var/cache/conftool/dbconfig/20220628-094414-ladsgroup.json |
[production] |
| 09:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1161 (re)pooling @ 10%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30549 and previous config saved to /var/cache/conftool/dbconfig/20220628-094321-root.json |
[production] |
| 09:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 10%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30548 and previous config saved to /var/cache/conftool/dbconfig/20220628-094237-root.json |
[production] |
| 09:38 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3316 (T298557)', diff saved to https://phabricator.wikimedia.org/P30547 and previous config saved to /var/cache/conftool/dbconfig/20220628-093807-marostegui.json |
[production] |
| 09:29 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1138', diff saved to https://phabricator.wikimedia.org/P30546 and previous config saved to /var/cache/conftool/dbconfig/20220628-092908-ladsgroup.json |
[production] |
| 09:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1161 (re)pooling @ 5%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30545 and previous config saved to /var/cache/conftool/dbconfig/20220628-092817-root.json |
[production] |
| 09:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 5%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30544 and previous config saved to /var/cache/conftool/dbconfig/20220628-092733-root.json |
[production] |
| 09:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1136 (re)pooling @ 100%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30543 and previous config saved to /var/cache/conftool/dbconfig/20220628-091649-root.json |
[production] |
| 09:14 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1138 (T309311)', diff saved to https://phabricator.wikimedia.org/P30542 and previous config saved to /var/cache/conftool/dbconfig/20220628-091403-ladsgroup.json |
[production] |
| 09:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1098:3316 (T298557)', diff saved to https://phabricator.wikimedia.org/P30541 and previous config saved to /var/cache/conftool/dbconfig/20220628-091318-marostegui.json |
[production] |
| 09:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1161 (re)pooling @ 2%: After kernel upgrade', diff saved to https://phabricator.wikimedia.org/P30540 and previous config saved to /var/cache/conftool/dbconfig/20220628-091313-root.json |
[production] |
| 09:13 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:13 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |