|
2020-11-16
ยง
|
| 17:48 |
<elukey> |
enable and run puppet on kafka-main2003 (it will start kafka services) - T267865 |
[production] |
| 17:42 |
<dwisehaupt> |
frmon1001 upgraded to buster |
[production] |
| 17:36 |
<volans> |
moved interfaces in Netbox from old to new switch - T267865 |
[production] |
| 17:24 |
<vgutierrez> |
switching back from lvs2010 to lvs2007 - T267865 |
[production] |
| 17:21 |
<vgutierrez> |
repooling cp2037 and cp2038 - T267865 |
[production] |
| 16:46 |
<elukey@cumin1001> |
END (FAIL) - Cookbook sre.hosts.decommission (exit_code=1) |
[production] |
| 16:40 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.decommission |
[production] |
| 16:16 |
<XioNoX> |
update c7 serial in row C VC config - T267865 |
[production] |
| 16:16 |
<rzl> |
disable puppet on A:mw-api-canary T264991 |
[production] |
| 16:14 |
<hnowlan@cumin1001> |
START - Cookbook sre.cassandra.roll-restart |
[production] |
| 16:08 |
<effie> |
disable puppet in appservers canaries to install ICU 63 - T264991 |
[production] |
| 16:07 |
<vgutierrez@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp2038.codfw.wmnet |
[production] |
| 16:07 |
<vgutierrez@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp2037.codfw.wmnet |
[production] |
| 16:06 |
<hnowlan@cumin1001> |
END (FAIL) - Cookbook sre.cassandra.roll-restart (exit_code=99) |
[production] |
| 16:03 |
<hnowlan> |
joined maps2006 to maps codfw cassandra cluster |
[production] |
| 16:01 |
<elukey@cumin1001> |
END (FAIL) - Cookbook sre.hosts.decommission (exit_code=1) |
[production] |
| 15:57 |
<hnowlan@cumin1001> |
START - Cookbook sre.cassandra.roll-restart |
[production] |
| 15:57 |
<hnowlan> |
roll-restarting eqiad restbase for java security updates |
[production] |
| 15:56 |
<hnowlan@cumin1001> |
END (PASS) - Cookbook sre.cassandra.roll-restart (exit_code=0) |
[production] |
| 15:50 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.decommission |
[production] |
| 15:40 |
<cdanis@cumin1001> |
END (PASS) - Cookbook sre.network.cf (exit_code=0) |
[production] |
| 15:40 |
<cdanis@cumin1001> |
START - Cookbook sre.network.cf |
[production] |
| 14:16 |
<hnowlan@cumin1001> |
START - Cookbook sre.cassandra.roll-restart |
[production] |
| 14:12 |
<marostegui@deploy1001> |
Synchronized wmf-config/db-eqiad.php: Repool pc1007 in pc1 after restarting mysql T266483 (duration: 00m 59s) |
[production] |
| 14:06 |
<marostegui> |
Restart pc1007's mysql T266483 |
[production] |
| 14:06 |
<marostegui@deploy1001> |
Synchronized wmf-config/db-eqiad.php: Depool pc1007 and place pc1010 instead of it T266483 (duration: 01m 00s) |
[production] |
| 13:23 |
<hnowlan@cumin1001> |
END (FAIL) - Cookbook sre.cassandra.roll-restart (exit_code=99) |
[production] |
| 13:00 |
<kormat> |
running schema change against s1 in codfw T259831 |
[production] |
| 12:59 |
<kormat@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 12:59 |
<kormat@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 12:43 |
<moritzm> |
installing tcpdump security updates |
[production] |
| 12:35 |
<hnowlan@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 12:35 |
<hnowlan@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 12:25 |
<hnowlan@cumin1001> |
START - Cookbook sre.cassandra.roll-restart |
[production] |
| 12:25 |
<hnowlan> |
roll-restarting restbase-codfw |
[production] |
| 12:24 |
<hnowlan@cumin1001> |
END (PASS) - Cookbook sre.cassandra.roll-restart (exit_code=0) |
[production] |
| 12:10 |
<hnowlan@cumin1001> |
END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) |
[production] |
| 12:10 |
<hnowlan@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 11:49 |
<hnowlan> |
roll restarting sessionstore for java updates |
[production] |
| 11:49 |
<hnowlan@cumin1001> |
START - Cookbook sre.cassandra.roll-restart |
[production] |
| 11:13 |
<moritzm> |
installing poppler security updates |
[production] |
| 10:46 |
<klausman@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 10:46 |
<klausman@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 10:45 |
<dcaro@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) |
[production] |
| 10:45 |
<dcaro@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 10:44 |
<dcaro@cumin1001> |
END (FAIL) - Cookbook sre.hosts.downtime (exit_code=99) |
[production] |
| 10:44 |
<dcaro@cumin1001> |
START - Cookbook sre.hosts.downtime |
[production] |
| 09:31 |
<gehel@cumin2001> |
END (FAIL) - Cookbook sre.elasticsearch.force-shard-allocation (exit_code=99) |
[production] |
| 09:31 |
<gehel@cumin2001> |
START - Cookbook sre.elasticsearch.force-shard-allocation |
[production] |
| 08:39 |
<godog> |
centrallog1001 move invalid config /etc/logrotate.d/logrotate-debug to /etc |
[production] |