|
2024-06-12
ยง
|
| 11:57 |
<claime> |
Manual restart of dump_cloud_ip_ranges.service on A:puppetserver and A:puppetmaster |
[production] |
| 11:55 |
<hnowlan@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mw-api-ext: apply |
[production] |
| 11:55 |
<hnowlan@deploy1002> |
helmfile [codfw] START helmfile.d/services/mw-api-ext: apply |
[production] |
| 11:54 |
<hnowlan@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-ext: apply |
[production] |
| 11:54 |
<hnowlan@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-api-ext: apply |
[production] |
| 11:54 |
<jmm@cumin2002> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1033.eqiad.wmnet |
[production] |
| 11:53 |
<jayme@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/eventstreams-internal: apply |
[production] |
| 11:53 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1034.eqiad.wmnet |
[production] |
| 11:53 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1034.eqiad.wmnet |
[production] |
| 11:53 |
<hnowlan@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: apply |
[production] |
| 11:53 |
<hnowlan@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-web: apply |
[production] |
| 11:52 |
<jayme@deploy1002> |
helmfile [codfw] START helmfile.d/services/eventstreams-internal: apply |
[production] |
| 11:52 |
<jayme@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/eventstreams: apply |
[production] |
| 11:51 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1212 (T367261)', diff saved to https://phabricator.wikimedia.org/P64695 and previous config saved to /var/cache/conftool/dbconfig/20240612-115143-marostegui.json |
[production] |
| 11:51 |
<jayme@deploy1002> |
helmfile [codfw] START helmfile.d/services/eventstreams: apply |
[production] |
| 11:51 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:51 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:51 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1212.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:51 |
<jayme@deploy1002> |
helmfile [staging] DONE helmfile.d/services/eventstreams-internal: apply |
[production] |
| 11:51 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1212.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:51 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1198 (T367261)', diff saved to https://phabricator.wikimedia.org/P64693 and previous config saved to /var/cache/conftool/dbconfig/20240612-115103-marostegui.json |
[production] |
| 11:50 |
<jayme@deploy1002> |
helmfile [staging] START helmfile.d/services/eventstreams-internal: apply |
[production] |
| 11:50 |
<jayme@deploy1002> |
helmfile [staging] DONE helmfile.d/services/eventstreams: apply |
[production] |
| 11:50 |
<jayme@deploy1002> |
helmfile [staging] START helmfile.d/services/eventstreams: apply |
[production] |
| 11:47 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db1191 (re)pooling @ 5%: Repooling', diff saved to https://phabricator.wikimedia.org/P64692 and previous config saved to /var/cache/conftool/dbconfig/20240612-114705-root.json |
[production] |
| 11:47 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti1034.eqiad.wmnet |
[production] |
| 11:46 |
<cgoubert@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/image-suggestion: apply |
[production] |
| 11:45 |
<jayme@deploy1002> |
helmfile [codfw] START helmfile.d/services/mw-mcrouter: apply |
[production] |
| 11:45 |
<cgoubert@deploy1002> |
helmfile [eqiad] START helmfile.d/services/image-suggestion: apply |
[production] |
| 11:45 |
<jayme@deploy1002> |
helmfile [staging] DONE helmfile.d/services/mw-mcrouter: apply |
[production] |
| 11:45 |
<jayme@deploy1002> |
helmfile [staging] START helmfile.d/services/mw-mcrouter: apply |
[production] |
| 11:44 |
<cgoubert@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/image-suggestion: apply |
[production] |
| 11:44 |
<cgoubert@deploy1002> |
helmfile [codfw] START helmfile.d/services/image-suggestion: apply |
[production] |
| 11:44 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depool db1191', diff saved to https://phabricator.wikimedia.org/P64691 and previous config saved to /var/cache/conftool/dbconfig/20240612-114410-root.json |
[production] |
| 11:42 |
<mvolz@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/citoid: apply |
[production] |
| 11:42 |
<mvolz@deploy1002> |
helmfile [eqiad] START helmfile.d/services/citoid: apply |
[production] |
| 11:39 |
<jayme@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-timeline: apply |
[production] |
| 11:38 |
<cgoubert@deploy1002> |
helmfile [staging] DONE helmfile.d/services/image-suggestion: apply |
[production] |
| 11:37 |
<jayme@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-timeline: apply |
[production] |
| 11:37 |
<jayme@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 11:37 |
<cgoubert@deploy1002> |
helmfile [staging] START helmfile.d/services/image-suggestion: apply |
[production] |
| 11:37 |
<mvolz@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/citoid: apply |
[production] |
| 11:37 |
<jayme@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 11:37 |
<jayme@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-media: apply |
[production] |
| 11:36 |
<jayme@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-media: apply |
[production] |
| 11:36 |
<jayme@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-constraints: apply |
[production] |
| 11:36 |
<jmm@cumin2002> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1034.eqiad.wmnet |
[production] |
| 11:36 |
<jayme@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-constraints: apply |
[production] |
| 11:36 |
<jayme@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox: apply |
[production] |
| 11:35 |
<mvolz@deploy1002> |
helmfile [codfw] START helmfile.d/services/citoid: apply |
[production] |