|
2023-05-17
ยง
|
| 10:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1220 (re)pooling @ 100%: Repooling after maintenance', diff saved to https://phabricator.wikimedia.org/P48303 and previous config saved to /var/cache/conftool/dbconfig/20230517-103129-root.json |
[production] |
| 10:29 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance es2026', diff saved to https://phabricator.wikimedia.org/P48302 and previous config saved to /var/cache/conftool/dbconfig/20230517-102948-ladsgroup.json |
[production] |
| 10:26 |
<oblivian@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-constraints: apply |
[production] |
| 10:25 |
<oblivian@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-constraints: apply |
[production] |
| 10:23 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'es2029 (re)pooling @ 75%: Maint over', diff saved to https://phabricator.wikimedia.org/P48301 and previous config saved to /var/cache/conftool/dbconfig/20230517-102310-ladsgroup.json |
[production] |
| 10:19 |
<oblivian@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/shellbox-constraints: apply |
[production] |
| 10:18 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/shellbox-constraints: apply |
[production] |
| 10:17 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/shellbox-constraints: apply |
[production] |
| 10:17 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/shellbox-constraints: apply |
[production] |
| 10:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1220 (re)pooling @ 75%: Repooling after maintenance', diff saved to https://phabricator.wikimedia.org/P48300 and previous config saved to /var/cache/conftool/dbconfig/20230517-101624-root.json |
[production] |
| 10:16 |
<oblivian@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 10:16 |
<oblivian@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 10:14 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance es2026', diff saved to https://phabricator.wikimedia.org/P48299 and previous config saved to /var/cache/conftool/dbconfig/20230517-101442-ladsgroup.json |
[production] |
| 10:09 |
<oblivian@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 10:08 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 10:08 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 10:08 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 10:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'es2029 (re)pooling @ 25%: Maint over', diff saved to https://phabricator.wikimedia.org/P48298 and previous config saved to /var/cache/conftool/dbconfig/20230517-100805-ladsgroup.json |
[production] |
| 10:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1220 (re)pooling @ 50%: Repooling after maintenance', diff saved to https://phabricator.wikimedia.org/P48297 and previous config saved to /var/cache/conftool/dbconfig/20230517-100120-root.json |
[production] |
| 09:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance es2026 (T335845)', diff saved to https://phabricator.wikimedia.org/P48296 and previous config saved to /var/cache/conftool/dbconfig/20230517-095936-ladsgroup.json |
[production] |
| 09:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling es2026 (T335845)', diff saved to https://phabricator.wikimedia.org/P48295 and previous config saved to /var/cache/conftool/dbconfig/20230517-095443-ladsgroup.json |
[production] |
| 09:54 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es2026.codfw.wmnet with reason: Maintenance |
[production] |
| 09:54 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es2026.codfw.wmnet with reason: Maintenance |
[production] |
| 09:53 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'es2029 (re)pooling @ 10%: Maint over', diff saved to https://phabricator.wikimedia.org/P48294 and previous config saved to /var/cache/conftool/dbconfig/20230517-095301-ladsgroup.json |
[production] |
| 09:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1220 (re)pooling @ 25%: Repooling after maintenance', diff saved to https://phabricator.wikimedia.org/P48293 and previous config saved to /var/cache/conftool/dbconfig/20230517-094615-root.json |
[production] |
| 09:39 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling es2029 (T335845)', diff saved to https://phabricator.wikimedia.org/P48292 and previous config saved to /var/cache/conftool/dbconfig/20230517-093928-ladsgroup.json |
[production] |
| 09:39 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on es2029.codfw.wmnet with reason: Maintenance |
[production] |
| 09:39 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on es2029.codfw.wmnet with reason: Maintenance |
[production] |
| 09:39 |
<elukey> |
roll restart pybal on lvs2010, lvs2009, lvs1020, lvs1019 to pick up a VIP (see https://gerrit.wikimedia.org/r/c/operations/puppet/+/920219) - T336726 |
[production] |
| 09:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1220 (re)pooling @ 10%: Repooling after maintenance', diff saved to https://phabricator.wikimedia.org/P48291 and previous config saved to /var/cache/conftool/dbconfig/20230517-093110-root.json |
[production] |
| 09:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1220 (re)pooling @ 5%: Repooling after maintenance', diff saved to https://phabricator.wikimedia.org/P48290 and previous config saved to /var/cache/conftool/dbconfig/20230517-091606-root.json |
[production] |
| 09:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1220 cleaning gtid_domain_id', diff saved to https://phabricator.wikimedia.org/P48289 and previous config saved to /var/cache/conftool/dbconfig/20230517-091407-root.json |
[production] |
| 08:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1132 (re)pooling @ 100%: Repooling after a crash', diff saved to https://phabricator.wikimedia.org/P48288 and previous config saved to /var/cache/conftool/dbconfig/20230517-085855-root.json |
[production] |
| 08:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1132 (re)pooling @ 75%: Repooling after a crash', diff saved to https://phabricator.wikimedia.org/P48287 and previous config saved to /var/cache/conftool/dbconfig/20230517-084350-root.json |
[production] |
| 08:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1132 (re)pooling @ 50%: Repooling after a crash', diff saved to https://phabricator.wikimedia.org/P48285 and previous config saved to /var/cache/conftool/dbconfig/20230517-082846-root.json |
[production] |
| 08:22 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host krb1001.eqiad.wmnet |
[production] |
| 08:17 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host krb1001.eqiad.wmnet |
[production] |
| 08:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1132 (re)pooling @ 25%: Repooling after a crash', diff saved to https://phabricator.wikimedia.org/P48284 and previous config saved to /var/cache/conftool/dbconfig/20230517-081341-root.json |
[production] |
| 08:08 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 08:08 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 08:05 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 08:04 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 07:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1132 (re)pooling @ 10%: Repooling after a crash', diff saved to https://phabricator.wikimedia.org/P48283 and previous config saved to /var/cache/conftool/dbconfig/20230517-075836-root.json |
[production] |
| 07:57 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 07:57 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/shellbox-syntaxhighlight: apply |
[production] |
| 07:48 |
<moritzm> |
upgrading krb1001 to Bullseye T331695 |
[production] |
| 07:45 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on krb1001.eqiad.wmnet with reason: Update to Bullseye |
[production] |
| 07:45 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on krb1001.eqiad.wmnet with reason: Update to Bullseye |
[production] |
| 07:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1132 (re)pooling @ 5%: Repooling after a crash', diff saved to https://phabricator.wikimedia.org/P48278 and previous config saved to /var/cache/conftool/dbconfig/20230517-074332-root.json |
[production] |
| 07:36 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'clear' for AS: 37468 |
[production] |