|
2022-03-22
ยง
|
| 12:19 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 12:19 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 12:18 |
<marostegui> |
dbmaint s6@eqiad T300992 |
[production] |
| 12:17 |
<marostegui> |
dbmaint s5@eqiad T300992 |
[production] |
| 12:16 |
<marostegui> |
dbmaint s8@eqiad T300992 |
[production] |
| 12:15 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 12:12 |
<ladsgroup@deploy1002> |
Synchronized wmf-config/InitialiseSettings.php: Config: [[gerrit:772816|Enable WRITE BOTH for templatelinks normalization in wikitech (T299421)]] (duration: 01m 41s) |
[production] |
| 12:11 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1100 (re)pooling @ 50%: After reboot', diff saved to https://phabricator.wikimedia.org/P22954 and previous config saved to /var/cache/conftool/dbconfig/20220322-121110-root.json |
[production] |
| 12:11 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1123 (re)pooling @ 50%: After reboot', diff saved to https://phabricator.wikimedia.org/P22953 and previous config saved to /var/cache/conftool/dbconfig/20220322-121101-root.json |
[production] |
| 12:01 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:01 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3317 (T298557)', diff saved to https://phabricator.wikimedia.org/P22952 and previous config saved to /var/cache/conftool/dbconfig/20220322-120123-marostegui.json |
[production] |
| 11:56 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1100 (re)pooling @ 25%: After reboot', diff saved to https://phabricator.wikimedia.org/P22951 and previous config saved to /var/cache/conftool/dbconfig/20220322-115606-root.json |
[production] |
| 11:55 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1123 (re)pooling @ 25%: After reboot', diff saved to https://phabricator.wikimedia.org/P22950 and previous config saved to /var/cache/conftool/dbconfig/20220322-115557-root.json |
[production] |
| 11:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3317', diff saved to https://phabricator.wikimedia.org/P22949 and previous config saved to /var/cache/conftool/dbconfig/20220322-114618-marostegui.json |
[production] |
| 11:41 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1100 (re)pooling @ 10%: After reboot', diff saved to https://phabricator.wikimedia.org/P22948 and previous config saved to /var/cache/conftool/dbconfig/20220322-114102-root.json |
[production] |
| 11:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1123 (re)pooling @ 10%: After reboot', diff saved to https://phabricator.wikimedia.org/P22946 and previous config saved to /var/cache/conftool/dbconfig/20220322-114051-root.json |
[production] |
| 11:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3317', diff saved to https://phabricator.wikimedia.org/P22945 and previous config saved to /var/cache/conftool/dbconfig/20220322-113113-marostegui.json |
[production] |
| 11:31 |
<marostegui> |
Reboot db1100 and db1123 for kernel upgrade before master swap |
[production] |
| 11:30 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1123 for reboot', diff saved to https://phabricator.wikimedia.org/P22944 and previous config saved to /var/cache/conftool/dbconfig/20220322-113003-marostegui.json |
[production] |
| 11:29 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1100 for reboot', diff saved to https://phabricator.wikimedia.org/P22943 and previous config saved to /var/cache/conftool/dbconfig/20220322-112931-marostegui.json |
[production] |
| 11:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3317 (T298557)', diff saved to https://phabricator.wikimedia.org/P22942 and previous config saved to /var/cache/conftool/dbconfig/20220322-111607-marostegui.json |
[production] |
| 11:10 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 11:03 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 11:03 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 10:56 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 10:46 |
<mmandere> |
pool cp1077 with HAProxy as TLS termination layer - T290005 |
[production] |
| 10:41 |
<mmandere@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cp1077.eqiad.wmnet with OS buster |
[production] |
| 10:26 |
<_joe_> |
running check-restart-php on api appservers |
[production] |
| 10:22 |
<_joe_> |
running check-and-restart on mw-eqiad-appservers |
[production] |
| 10:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1098:3317 (T298557)', diff saved to https://phabricator.wikimedia.org/P22940 and previous config saved to /var/cache/conftool/dbconfig/20220322-101354-marostegui.json |
[production] |
| 10:13 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:13 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317 (T298557)', diff saved to https://phabricator.wikimedia.org/P22939 and previous config saved to /var/cache/conftool/dbconfig/20220322-101346-marostegui.json |
[production] |
| 10:03 |
<jnuche@deploy1002> |
rebuilt and synchronized wikiversions files: group0 wikis to 1.39.0-wmf.3 refs T300203 |
[production] |
| 09:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P22938 and previous config saved to /var/cache/conftool/dbconfig/20220322-095841-marostegui.json |
[production] |
| 09:54 |
<mmandere@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cp1077.eqiad.wmnet with reason: host reimage |
[production] |
| 09:54 |
<jnuche@deploy1002> |
Finished scap: testwikis wikis to 1.39.0-wmf.3 refs T300203 (duration: 62m 07s) |
[production] |
| 09:51 |
<mmandere@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cp1077.eqiad.wmnet with reason: host reimage |
[production] |
| 09:46 |
<dcaro@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on cloudcontrol1005.wikimedia.org with reason: dcaro testing backups |
[production] |
| 09:46 |
<dcaro@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on cloudcontrol1005.wikimedia.org with reason: dcaro testing backups |
[production] |
| 09:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P22937 and previous config saved to /var/cache/conftool/dbconfig/20220322-094335-marostegui.json |
[production] |
| 09:34 |
<mmandere@cumin1001> |
START - Cookbook sre.hosts.reimage for host cp1077.eqiad.wmnet with OS buster |
[production] |
| 09:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317 (T298557)', diff saved to https://phabricator.wikimedia.org/P22936 and previous config saved to /var/cache/conftool/dbconfig/20220322-092830-marostegui.json |
[production] |
| 09:25 |
<mmandere> |
depool cp1077 for reimage - T290005 |
[production] |
| 09:17 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1175 (re)pooling @ 100%: After reimage', diff saved to https://phabricator.wikimedia.org/P22935 and previous config saved to /var/cache/conftool/dbconfig/20220322-091718-root.json |
[production] |
| 09:11 |
<dcausse> |
restarted blazegraph on wdqs2002 (deadlocked) |
[production] |
| 09:02 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1175 (re)pooling @ 75%: After reimage', diff saved to https://phabricator.wikimedia.org/P22934 and previous config saved to /var/cache/conftool/dbconfig/20220322-090214-root.json |
[production] |
| 08:59 |
<XioNoX> |
drmrs propagate LVS med to core routers |
[production] |
| 08:52 |
<jnuche@deploy1002> |
Started scap: testwikis wikis to 1.39.0-wmf.3 refs T300203 |
[production] |