|
2025-05-26
ยง
|
| 17:37 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2178 (T395241)', diff saved to https://phabricator.wikimedia.org/P76438 and previous config saved to /var/cache/conftool/dbconfig/20250526-173700-fceratto.json |
[production] |
| 17:29 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db2178 (T395241)', diff saved to https://phabricator.wikimedia.org/P76437 and previous config saved to /var/cache/conftool/dbconfig/20250526-172912-fceratto.json |
[production] |
| 17:29 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2178.codfw.wmnet with reason: Maintenance |
[production] |
| 17:28 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2171 (T395241)', diff saved to https://phabricator.wikimedia.org/P76436 and previous config saved to /var/cache/conftool/dbconfig/20250526-172844-fceratto.json |
[production] |
| 17:13 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2171', diff saved to https://phabricator.wikimedia.org/P76435 and previous config saved to /var/cache/conftool/dbconfig/20250526-171338-fceratto.json |
[production] |
| 16:58 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2171', diff saved to https://phabricator.wikimedia.org/P76434 and previous config saved to /var/cache/conftool/dbconfig/20250526-165831-fceratto.json |
[production] |
| 16:43 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2171 (T395241)', diff saved to https://phabricator.wikimedia.org/P76433 and previous config saved to /var/cache/conftool/dbconfig/20250526-164324-fceratto.json |
[production] |
| 16:35 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db2171 (T395241)', diff saved to https://phabricator.wikimedia.org/P76432 and previous config saved to /var/cache/conftool/dbconfig/20250526-163530-fceratto.json |
[production] |
| 16:35 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2171.codfw.wmnet with reason: Maintenance |
[production] |
| 16:35 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2157 (T395241)', diff saved to https://phabricator.wikimedia.org/P76431 and previous config saved to /var/cache/conftool/dbconfig/20250526-163502-fceratto.json |
[production] |
| 16:19 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2157', diff saved to https://phabricator.wikimedia.org/P76430 and previous config saved to /var/cache/conftool/dbconfig/20250526-161955-fceratto.json |
[production] |
| 16:15 |
<jgiannelos@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/changeprop: apply |
[production] |
| 16:14 |
<jgiannelos@deploy1003> |
helmfile [eqiad] START helmfile.d/services/changeprop: apply |
[production] |
| 16:14 |
<jgiannelos@deploy1003> |
helmfile [staging] DONE helmfile.d/services/changeprop: apply |
[production] |
| 16:13 |
<jgiannelos@deploy1003> |
helmfile [staging] START helmfile.d/services/changeprop: apply |
[production] |
| 16:07 |
<cgoubert@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/mw-cron: apply |
[production] |
| 16:07 |
<cgoubert@deploy1003> |
helmfile [eqiad] START helmfile.d/services/mw-cron: apply |
[production] |
| 16:04 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2157', diff saved to https://phabricator.wikimedia.org/P76429 and previous config saved to /var/cache/conftool/dbconfig/20250526-160447-fceratto.json |
[production] |
| 15:49 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2157 (T395241)', diff saved to https://phabricator.wikimedia.org/P76428 and previous config saved to /var/cache/conftool/dbconfig/20250526-154939-fceratto.json |
[production] |
| 15:49 |
<jgiannelos@deploy1003> |
helmfile [codfw] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:48 |
<jgiannelos@deploy1003> |
helmfile [codfw] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:48 |
<jgiannelos@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:47 |
<jgiannelos@deploy1003> |
helmfile [eqiad] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:47 |
<jgiannelos@deploy1003> |
helmfile [staging] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:47 |
<jgiannelos@deploy1003> |
helmfile [staging] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:46 |
<jgiannelos@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:46 |
<jgiannelos@deploy1003> |
helmfile [eqiad] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:46 |
<volans@cumin1003> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts cirrussearch2111.codfw.wmnet |
[production] |
| 15:45 |
<volans@cumin1003> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=1) for host cirrussearch2111.codfw.wmnet |
[production] |
| 15:45 |
<jgiannelos@deploy1003> |
helmfile [staging] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:44 |
<jgiannelos@deploy1003> |
helmfile [staging] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:43 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db2157 (T395241)', diff saved to https://phabricator.wikimedia.org/P76427 and previous config saved to /var/cache/conftool/dbconfig/20250526-154304-fceratto.json |
[production] |
| 15:42 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2157.codfw.wmnet with reason: Maintenance |
[production] |
| 15:32 |
<hnowlan@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 15:31 |
<hnowlan@deploy1003> |
helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 15:29 |
<hnowlan@deploy1003> |
helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 15:29 |
<volans@cumin1003> |
START - Cookbook sre.hosts.reboot-single for host cirrussearch2111.codfw.wmnet |
[production] |
| 15:28 |
<hnowlan@deploy1003> |
helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 15:28 |
<volans@cumin1003> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts cirrussearch2111.codfw.wmnet |
[production] |
| 15:15 |
<hnowlan@deploy1003> |
helmfile [staging] DONE helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 15:15 |
<hnowlan@deploy1003> |
helmfile [staging] START helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 15:12 |
<hnowlan@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/changeprop: apply |
[production] |
| 15:12 |
<hnowlan@deploy1003> |
helmfile [eqiad] START helmfile.d/services/changeprop: apply |
[production] |
| 15:10 |
<jgiannelos@deploy1003> |
helmfile [staging] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:10 |
<jgiannelos@deploy1003> |
helmfile [staging] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:10 |
<fabfur@cumin1002> |
conftool action : set/pooled=yes; selector: name=cp7001.magru.wmnet |
[production] |
| 15:10 |
<jgiannelos@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/mobileapps: apply |
[production] |
| 15:10 |
<hnowlan@deploy1003> |
helmfile [codfw] DONE helmfile.d/services/changeprop: apply |
[production] |
| 15:10 |
<jgiannelos@deploy1003> |
helmfile [eqiad] START helmfile.d/services/mobileapps: apply |
[production] |
| 15:09 |
<hnowlan@deploy1003> |
helmfile [codfw] START helmfile.d/services/changeprop: apply |
[production] |