|
2021-02-24
ยง
|
| 14:04 |
<pt1979@cumin2001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2150.codfw.wmnet with reason: REIMAGE |
[production] |
| 13:46 |
<marostegui> |
Compare data between db1134 and db1163 T275343 |
[production] |
| 13:34 |
<moritzm> |
restarting FPM/mcrouter on mw canaries to pick up openssl updates |
[production] |
| 13:11 |
<moritzm> |
installing openssl security updates on buster |
[production] |
| 12:32 |
<Urbanecm> |
Two undeployed patches were reverted to unbreak deployments (666340, 666341), cc marxarelli |
[production] |
| 12:25 |
<phuedx@deploy1001> |
Synchronized php-1.36.0-wmf.32/extensions/WikimediaEvents: Backport: [[gerrit:666339|Fix dynamically loaded instruments]] (duration: 01m 11s) |
[production] |
| 12:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1112 (re)pooling @ 100%: After schema change', diff saved to https://phabricator.wikimedia.org/P14465 and previous config saved to /var/cache/conftool/dbconfig/20210224-122043-root.json |
[production] |
| 12:18 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'changeprop-jobqueue' for release 'production' . |
[production] |
| 12:17 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'changeprop' for release 'production' . |
[production] |
| 12:17 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'changeprop-jobqueue' for release 'production' . |
[production] |
| 12:16 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'changeprop' for release 'production' . |
[production] |
| 12:15 |
<jayme@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'changeprop-jobqueue' for release 'staging' . |
[production] |
| 12:14 |
<jayme@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'changeprop' for release 'staging' . |
[production] |
| 12:06 |
<hnowlan> |
restarting mtail on A:mw-api or A:parsoid or A:mw-jobrunner or A:mw |
[production] |
| 12:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1112 (re)pooling @ 75%: After schema change', diff saved to https://phabricator.wikimedia.org/P14464 and previous config saved to /var/cache/conftool/dbconfig/20210224-120538-root.json |
[production] |
| 11:54 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'zotero' for release 'production' . |
[production] |
| 11:53 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'termbox' for release 'production' . |
[production] |
| 11:51 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'similar-users' for release 'main' . |
[production] |
| 11:50 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1112 (re)pooling @ 50%: After schema change', diff saved to https://phabricator.wikimedia.org/P14463 and previous config saved to /var/cache/conftool/dbconfig/20210224-115034-root.json |
[production] |
| 11:45 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'recommendation-api' for release 'production' . |
[production] |
| 11:44 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'push-notifications' for release 'main' . |
[production] |
| 11:42 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'proton' for release 'production' . |
[production] |
| 11:39 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mobileapps' for release 'production' . |
[production] |
| 11:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1112 (re)pooling @ 25%: After schema change', diff saved to https://phabricator.wikimedia.org/P14462 and previous config saved to /var/cache/conftool/dbconfig/20210224-113531-root.json |
[production] |
| 11:33 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mathoid' for release 'production' . |
[production] |
| 11:32 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventstreams-internal' for release 'main' . |
[production] |
| 11:29 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventstreams' for release 'production' . |
[production] |
| 11:29 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventstreams' for release 'canary' . |
[production] |
| 11:24 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-main' for release 'production' . |
[production] |
| 11:23 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-main' for release 'canary' . |
[production] |
| 11:22 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'production' . |
[production] |
| 11:22 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'canary' . |
[production] |
| 11:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1112 (re)pooling @ 10%: After schema change', diff saved to https://phabricator.wikimedia.org/P14461 and previous config saved to /var/cache/conftool/dbconfig/20210224-112027-root.json |
[production] |
| 11:20 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'eventgate-analytics' for release 'production' . |
[production] |
| 11:15 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'cxserver' for release 'production' . |
[production] |
| 11:14 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'citoid' for release 'production' . |
[production] |
| 11:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1112 for schema change', diff saved to https://phabricator.wikimedia.org/P14460 and previous config saved to /var/cache/conftool/dbconfig/20210224-111301-marostegui.json |
[production] |
| 11:12 |
<jayme@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'apertium' for release 'production' . |
[production] |
| 11:06 |
<aborrero@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudmetrics1002.eqiad.wmnet with reason: serer issue |
[production] |
| 11:06 |
<aborrero@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudmetrics1002.eqiad.wmnet with reason: serer issue |
[production] |
| 10:52 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 100%: After schema change', diff saved to https://phabricator.wikimedia.org/P14459 and previous config saved to /var/cache/conftool/dbconfig/20210224-105204-root.json |
[production] |
| 10:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 75%: After schema change', diff saved to https://phabricator.wikimedia.org/P14458 and previous config saved to /var/cache/conftool/dbconfig/20210224-103700-root.json |
[production] |
| 10:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 50%: After schema change', diff saved to https://phabricator.wikimedia.org/P14457 and previous config saved to /var/cache/conftool/dbconfig/20210224-102157-root.json |
[production] |
| 10:20 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'zotero' for release 'production' . |
[production] |
| 10:19 |
<moritzm> |
installing gnutls28 bugfix updates from Buster 10.8 point release |
[production] |
| 10:16 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'termbox' for release 'production' . |
[production] |
| 10:13 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'similar-users' for release 'main' . |
[production] |
| 10:10 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'recommendation-api' for release 'production' . |
[production] |
| 10:06 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1157 (re)pooling @ 25%: After schema change', diff saved to https://phabricator.wikimedia.org/P14456 and previous config saved to /var/cache/conftool/dbconfig/20210224-100653-root.json |
[production] |
| 10:04 |
<jayme@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'push-notifications' for release 'main' . |
[production] |