|
2020-09-03
ยง
|
| 17:45 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 17:44 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 17:43 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 17:43 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 17:41 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 17:36 |
<mholloway-shell@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'mobileapps' for release 'nontls' . |
[production] |
| 17:36 |
<mholloway-shell@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'mobileapps' for release 'production' . |
[production] |
| 17:32 |
<mholloway-shell@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mobileapps' for release 'production' . |
[production] |
| 17:32 |
<mholloway-shell@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'mobileapps' for release 'nontls' . |
[production] |
| 17:28 |
<mholloway-shell@deploy1001> |
helmfile [staging] Ran 'sync' command on namespace 'mobileapps' for release 'staging' . |
[production] |
| 17:19 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 17:16 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 17:02 |
<papaul> |
power down ores2009 for DIMM upgrade |
[production] |
| 16:45 |
<papaul> |
power down ores2008 for DIMM upgrade |
[production] |
| 16:33 |
<papaul> |
power down ores2007 for DIMM upgrade |
[production] |
| 16:24 |
<elukey> |
roll restart aqs on aqs1* to pick up new druid settings |
[production] |
| 16:05 |
<papaul> |
power down ores2006 for DIMM upgrade |
[production] |
| 15:51 |
<papaul> |
power down ores2005 for DIMM upgrade |
[production] |
| 15:33 |
<papaul> |
power down ores2004 for DIMM upgrade |
[production] |
| 15:30 |
<moritzm> |
installing nginx updates on apt* and htmldumper1001 |
[production] |
| 15:25 |
<moritzm> |
installing firejail update (along with restarts) on thumbor1001, maps1001, restbase1016 (and -dev) |
[production] |
| 15:21 |
<papaul> |
power down ores2003 for DIMM upgrade |
[production] |
| 15:17 |
<moritzm> |
installing firejail security updates on parsoid servers |
[production] |
| 15:08 |
<papaul> |
power down ores2002 for DIMM upgrade |
[production] |
| 14:53 |
<papaul> |
power down ores2001 for DIMM upgrade |
[production] |
| 14:36 |
<hnowlan@deploy1001> |
helmfile [codfw] Ran 'sync' command on namespace 'api-gateway' for release 'production' . |
[production] |
| 14:30 |
<hnowlan@deploy1001> |
helmfile [eqiad] Ran 'sync' command on namespace 'api-gateway' for release 'production' . |
[production] |
| 14:29 |
<jmm@deploy1001> |
Finished deploy [debmonitor/deploy@fb64c52]: deploy to new buster host (duration: 00m 06s) |
[production] |
| 14:29 |
<jmm@deploy1001> |
Started deploy [debmonitor/deploy@fb64c52]: deploy to new buster host |
[production] |
| 14:13 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 14:11 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 14:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Set wikitech back to RW after maintenance T260324', diff saved to https://phabricator.wikimedia.org/P12490 and previous config saved to /var/cache/conftool/dbconfig/20200903-140451-marostegui.json |
[production] |
| 14:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Promote db1128 to wikitech master T260324', diff saved to https://phabricator.wikimedia.org/P12489 and previous config saved to /var/cache/conftool/dbconfig/20200903-140436-marostegui.json |
[production] |
| 14:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Promote db1128 to wikitech master T260324', diff saved to https://phabricator.wikimedia.org/P12488 and previous config saved to /var/cache/conftool/dbconfig/20200903-140411-marostegui.json |
[production] |
| 14:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Set wikitech as read-only for maintenance T260324', diff saved to https://phabricator.wikimedia.org/P12487 and previous config saved to /var/cache/conftool/dbconfig/20200903-140135-marostegui.json |
[production] |
| 14:00 |
<marostegui> |
Failover m5 (wikitech) master - T260324 |
[production] |
| 13:53 |
<jmm@cumin2001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 13:53 |
<jmm@cumin2001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 13:43 |
<jmm@deploy1001> |
Finished deploy [debmonitor/deploy@fb64c52]: deploy to new buster host (duration: 00m 18s) |
[production] |
| 13:43 |
<jmm@deploy1001> |
Started deploy [debmonitor/deploy@fb64c52]: deploy to new buster host |
[production] |
| 13:40 |
<jmm@deploy1001> |
Finished deploy [debmonitor/deploy@25dbd20]: deploy to new buster host, now the --force is with me (duration: 01m 29s) |
[production] |
| 13:39 |
<jmm@deploy1001> |
Started deploy [debmonitor/deploy@25dbd20]: deploy to new buster host, now the --force is with me |
[production] |
| 13:32 |
<jmm@deploy1001> |
Finished deploy [debmonitor/deploy@25dbd20]: deploy to new buster host (duration: 00m 05s) |
[production] |
| 13:32 |
<jmm@deploy1001> |
Started deploy [debmonitor/deploy@25dbd20]: deploy to new buster host |
[production] |
| 13:08 |
<marostegui> |
Start pre m5 failover steps T260324 |
[production] |
| 12:46 |
<marostegui> |
Deploy MCR schema change on s7 eqiad master (lag might show up) - T238966 |
[production] |
| 12:30 |
<hnowlan> |
enabling puppet on appservers, finished rollout of api.wikimedia.org https://gerrit.wikimedia.org/r/c/operations/puppet/+/623833 |
[production] |
| 12:19 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'Shift weights in s2 codfw to account for db2125 being down T260670', diff saved to https://phabricator.wikimedia.org/P12485 and previous config saved to /var/cache/conftool/dbconfig/20200903-121916-kormat.json |
[production] |
| 12:17 |
<moritzm> |
installing openexr security updates for stretch |
[production] |
| 12:03 |
<kormat@cumin1001> |
dbctl commit (dc=all): 'Depool db2125 after hw issue', diff saved to https://phabricator.wikimedia.org/P12483 and previous config saved to /var/cache/conftool/dbconfig/20200903-120304-kormat.json |
[production] |