|
2021-02-24
ยง
|
| 12:17 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'changeprop-jobqueue' for release 'production' . |
[production] |
| 12:16 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'changeprop' for release 'production' . |
[production] |
| 12:15 |
<jayme@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'changeprop-jobqueue' for release 'staging' . |
[production] |
| 12:14 |
<jayme@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'changeprop' for release 'staging' . |
[production] |
| 12:06 |
<hnowlan> |
restarting mtail on A:mw-api or A:parsoid or A:mw-jobrunner or A:mw |
[production] |
| 12:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1112 (re)pooling @ 75%: After schema change', diff saved to https://phabricator.wikimedia.org/P14464 and previous config saved to /var/cache/conftool/dbconfig/20210224-120538-root.json |
[production] |
| 11:54 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'zotero' for release 'production' . |
[production] |
| 11:53 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'termbox' for release 'production' . |
[production] |
| 11:51 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'similar-users' for release 'main' . |
[production] |
| 11:50 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1112 (re)pooling @ 50%: After schema change', diff saved to https://phabricator.wikimedia.org/P14463 and previous config saved to /var/cache/conftool/dbconfig/20210224-115034-root.json |
[production] |
| 11:45 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'recommendation-api' for release 'production' . |
[production] |
| 11:44 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'push-notifications' for release 'main' . |
[production] |
| 11:42 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'proton' for release 'production' . |
[production] |
| 11:39 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mobileapps' for release 'production' . |
[production] |
| 11:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1112 (re)pooling @ 25%: After schema change', diff saved to https://phabricator.wikimedia.org/P14462 and previous config saved to /var/cache/conftool/dbconfig/20210224-113531-root.json |
[production] |
| 11:33 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mathoid' for release 'production' . |
[production] |
| 11:32 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventstreams-internal' for release 'main' . |
[production] |
| 11:29 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventstreams' for release 'production' . |
[production] |
| 11:29 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventstreams' for release 'canary' . |
[production] |
| 11:24 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-main' for release 'production' . |
[production] |
| 11:23 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-main' for release 'canary' . |
[production] |
| 11:22 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'production' . |
[production] |
| 11:22 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'canary' . |
[production] |
| 11:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1112 (re)pooling @ 10%: After schema change', diff saved to https://phabricator.wikimedia.org/P14461 and previous config saved to /var/cache/conftool/dbconfig/20210224-112027-root.json |
[production] |
| 11:20 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-analytics' for release 'production' . |
[production] |
| 11:15 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'cxserver' for release 'production' . |
[production] |
| 11:14 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'citoid' for release 'production' . |
[production] |
| 11:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1112 for schema change', diff saved to https://phabricator.wikimedia.org/P14460 and previous config saved to /var/cache/conftool/dbconfig/20210224-111301-marostegui.json |
[production] |
| 11:12 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'apertium' for release 'production' . |
[production] |
| 11:06 |
<aborrero@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudmetrics1002.eqiad.wmnet with reason: serer issue |
[production] |
| 11:06 |
<aborrero@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudmetrics1002.eqiad.wmnet with reason: serer issue |
[production] |
| 10:52 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 100%: After schema change', diff saved to https://phabricator.wikimedia.org/P14459 and previous config saved to /var/cache/conftool/dbconfig/20210224-105204-root.json |
[production] |
| 10:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 75%: After schema change', diff saved to https://phabricator.wikimedia.org/P14458 and previous config saved to /var/cache/conftool/dbconfig/20210224-103700-root.json |
[production] |
| 10:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 50%: After schema change', diff saved to https://phabricator.wikimedia.org/P14457 and previous config saved to /var/cache/conftool/dbconfig/20210224-102157-root.json |
[production] |
| 10:20 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'zotero' for release 'production' . |
[production] |
| 10:19 |
<moritzm> |
installing gnutls28 bugfix updates from Buster 10.8 point release |
[production] |
| 10:16 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'termbox' for release 'production' . |
[production] |
| 10:13 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'similar-users' for release 'main' . |
[production] |
| 10:10 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'recommendation-api' for release 'production' . |
[production] |
| 10:06 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 25%: After schema change', diff saved to https://phabricator.wikimedia.org/P14456 and previous config saved to /var/cache/conftool/dbconfig/20210224-100653-root.json |
[production] |
| 10:04 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'push-notifications' for release 'main' . |
[production] |
| 10:02 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'proton' for release 'production' . |
[production] |
| 09:56 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'mobileapps' for release 'production' . |
[production] |
| 09:51 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 10%: After schema change', diff saved to https://phabricator.wikimedia.org/P14455 and previous config saved to /var/cache/conftool/dbconfig/20210224-095150-root.json |
[production] |
| 09:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1157 for schema change', diff saved to https://phabricator.wikimedia.org/P14454 and previous config saved to /var/cache/conftool/dbconfig/20210224-094523-marostegui.json |
[production] |
| 09:34 |
<marostegui> |
Update pc2007, pc2010, db2071 |
[production] |
| 09:31 |
<marostegui> |
Update db1077 |
[production] |
| 09:27 |
<jiji@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host mc1033.eqiad.wmnet |
[production] |
| 09:20 |
<jiji@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host mc1033.eqiad.wmnet |
[production] |
| 09:19 |
<effie> |
upgrade memcached on mc1033, mc2033 |
[production] |