|
2021-06-29
ยง
|
| 15:15 |
<mutante> |
[mwlog2002:~] $ sudo systemctl start mw-log-cleanup |
[production] |
| 15:06 |
<dcausse> |
banning elastic2054 |
[production] |
| 14:53 |
<dzahn@cumin1001> |
conftool action : set/weight=1; selector: name=mw227[1-2].codfw.wmnet,service=canary |
[production] |
| 14:52 |
<dzahn@cumin1001> |
conftool action : set/weight=1; selector: name=mw227[8-9].codfw.wmnet,service=canary |
[production] |
| 14:52 |
<dzahn@cumin1001> |
conftool action : set/weight=1; selector: name=mw225[1-2].codfw.wmnet,service=canary |
[production] |
| 14:52 |
<effie> |
depool mw2383 as it is misbehaving |
[production] |
| 14:47 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.08-update-tendril (exit_code=0) |
[production] |
| 14:47 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.08-update-tendril |
[production] |
| 14:47 |
<dzahn@cumin1001> |
conftool action : set/weight=25; selector: name=mw226[1-2].codfw.wmnet |
[production] |
| 14:47 |
<dzahn@cumin1001> |
conftool action : set/weight=25; selector: name=mw2290.codfw.wmnet |
[production] |
| 14:46 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.08-start-maintenance (exit_code=0) |
[production] |
| 14:46 |
<dzahn@cumin1001> |
conftool action : set/weight=25; selector: name=mw22[7-8][0-9].codfw.wmnet |
[production] |
| 14:45 |
<dzahn@cumin1001> |
conftool action : set/weight=25; selector: name=mw225[1-8].codfw.wmnet |
[production] |
| 14:44 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.08-start-maintenance |
[production] |
| 14:44 |
<dzahn@cumin1001> |
conftool action : set/weight=25; selector: name=mw225[1-8].codfw.wmnet,service=api_appserver |
[production] |
| 14:43 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.08-run-puppet-on-db-masters (exit_code=0) |
[production] |
| 14:38 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.08-run-puppet-on-db-masters |
[production] |
| 14:38 |
<_joe_> |
restarting pohp-fpm on mw2383 |
[production] |
| 14:38 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.08-restore-ttl (exit_code=0) |
[production] |
| 14:37 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.08-restore-ttl |
[production] |
| 14:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Reduce db2103 (s1) weight a bit', diff saved to https://phabricator.wikimedia.org/P16739 and previous config saved to /var/cache/conftool/dbconfig/20210629-143742-marostegui.json |
[production] |
| 14:37 |
<_joe_> |
repooling mw2383 |
[production] |
| 14:36 |
<_joe_> |
depooling mw2383 |
[production] |
| 14:30 |
<legoktm@deploy1002> |
Synchronized wmf-config/db-codfw.php: fix trwikivoyage (duration: 01m 01s) |
[production] |
| 14:29 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.08-restart-envoy-on-jobrunners (exit_code=0) |
[production] |
| 14:29 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.08-restart-envoy-on-jobrunners |
[production] |
| 14:28 |
<Krinkle> |
TODO: Don't duplicate `sectionsByDB` between db-* files |
[production] |
| 14:23 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.07-set-readwrite (exit_code=0) |
[production] |
| 14:23 |
<jayme@cumin1001> |
MediaWiki read-only period ends at: 2021-06-29 14:23:23.504447 |
[production] |
| 14:23 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.07-set-readwrite |
[production] |
| 14:23 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.06-set-db-readwrite (exit_code=0) |
[production] |
| 14:23 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.06-set-db-readwrite |
[production] |
| 14:23 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.05-invert-redis-sessions (exit_code=0) |
[production] |
| 14:22 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.05-invert-redis-sessions |
[production] |
| 14:22 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.04-switch-mediawiki (exit_code=0) |
[production] |
| 14:22 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.04-switch-mediawiki |
[production] |
| 14:22 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.03-set-db-readonly (exit_code=0) |
[production] |
| 14:21 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.03-set-db-readonly |
[production] |
| 14:21 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.02-set-readonly (exit_code=0) |
[production] |
| 14:21 |
<jayme@cumin1001> |
MediaWiki read-only period starts at: 2021-06-29 14:21:26.671853 |
[production] |
| 14:21 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.02-set-readonly |
[production] |
| 14:15 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.01-stop-maintenance (exit_code=0) |
[production] |
| 14:15 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.01-stop-maintenance |
[production] |
| 14:13 |
<kormat@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 44 hosts with reason: DC switchover |
[production] |
| 14:13 |
<kormat@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 44 hosts with reason: DC switchover |
[production] |
| 14:12 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.00-warmup-caches (exit_code=0) |
[production] |
| 14:11 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.00-warmup-caches |
[production] |
| 14:10 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.00-warmup-caches (exit_code=0) |
[production] |
| 14:09 |
<jayme@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.00-warmup-caches |
[production] |
| 14:08 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.00-reduce-ttl (exit_code=0) |
[production] |