|
2021-09-27
ยง
|
| 14:11 |
<otto@deploy1002> |
helmfile [codfw] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'canary' . |
[production] |
| 14:11 |
<otto@deploy1002> |
helmfile [codfw] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'production' . |
[production] |
| 13:59 |
<otto@puppetmaster1001> |
conftool action : set/pooled=false; selector: dnsdisc=eventgate-logging-external,name=codfw |
[production] |
| 13:58 |
<ottomata> |
beginning re-deploy of eventgate-logging-external - https://phabricator.wikimedia.org/T291504#7380252 |
[production] |
| 13:57 |
<otto@puppetmaster1001> |
conftool action : set/ttl=10; selector: dnsdisc=eventgate-logging-external |
[production] |
| 13:52 |
<oblivian@deploy1002> |
helmfile [staging] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'production' . |
[production] |
| 13:48 |
<otto@deploy1002> |
helmfile [staging] Ran 'sync' command on namespace 'eventgate-logging-external' for release 'production' . |
[production] |
| 13:36 |
<mbsantos@deploy1002> |
Finished deploy [kartotherian/deploy@04d2df4]: tegola: use eqiad discovery endpoin (duration: 00m 15s) |
[production] |
| 13:35 |
<mbsantos@deploy1002> |
Started deploy [kartotherian/deploy@04d2df4]: tegola: use eqiad discovery endpoin |
[production] |
| 11:45 |
<marostegui> |
Upgrade es4 in codfw to 10.4.21 |
[production] |
| 11:43 |
<marostegui> |
Turn off es2021 for onsite maintenance T290327 |
[production] |
| 11:09 |
<volans> |
re-enabled puppet on install hosts after deployment of g/723996 - T221388 |
[production] |
| 11:02 |
<volans> |
disabling puppet on install hosts to deploy 723996 - T221388 |
[production] |
| 10:35 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host thanos-fe2003.codfw.wmnet |
[production] |
| 10:29 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host thanos-fe2003.codfw.wmnet |
[production] |
| 10:02 |
<jmm@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cumin2002.codfw.wmnet |
[production] |
| 09:55 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host thanos-fe2002.codfw.wmnet |
[production] |
| 09:53 |
<jmm@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cumin2002.codfw.wmnet |
[production] |
| 09:51 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host thanos-fe2002.codfw.wmnet |
[production] |
| 09:50 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host thanos-fe2001.codfw.wmnet |
[production] |
| 09:45 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host mx2001.wikimedia.org |
[production] |
| 09:44 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host thanos-fe2001.codfw.wmnet |
[production] |
| 09:43 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host thanos-fe1003.eqiad.wmnet |
[production] |
| 09:42 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host mx2001.wikimedia.org |
[production] |
| 09:38 |
<marostegui> |
Optimize table commonswiki.image on codfw (s4 will show lag) - T288273 |
[production] |
| 09:38 |
<aborrero@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cloudcontrol1004.wikimedia.org |
[production] |
| 09:37 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host thanos-fe1003.eqiad.wmnet |
[production] |
| 09:36 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host thanos-fe1002.eqiad.wmnet |
[production] |
| 09:34 |
<jbond@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on ldap-replica2006.wikimedia.org with reason: reboot - T291813 |
[production] |
| 09:33 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.downtime for 0:10:00 on ldap-replica2006.wikimedia.org with reason: reboot - T291813 |
[production] |
| 09:31 |
<jbond@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on ldap-replica2005.wikimedia.org with reason: reboot - T291813 |
[production] |
| 09:30 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.downtime for 0:10:00 on ldap-replica2005.wikimedia.org with reason: reboot - T291813 |
[production] |
| 09:30 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host thanos-fe1002.eqiad.wmnet |
[production] |
| 09:29 |
<moritzm> |
systemctl reset-failed networking T273026 |
[production] |
| 09:29 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host thanos-fe1001.eqiad.wmnet |
[production] |
| 09:27 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on ldap-replica1004.wikimedia.org with reason: reboot - T291813 |
[production] |
| 09:27 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.downtime for 0:10:00 on ldap-replica1004.wikimedia.org with reason: reboot - T291813 |
[production] |
| 09:27 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host mx1001.wikimedia.org |
[production] |
| 09:24 |
<aborrero@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cloudcontrol1004.wikimedia.org |
[production] |
| 09:23 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host thanos-fe1001.eqiad.wmnet |
[production] |
| 09:22 |
<filippo@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=99) for host thanos-fe1001.eqiad.wmnet |
[production] |
| 09:22 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host thanos-fe1001.eqiad.wmnet |
[production] |
| 09:18 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host mx1001.wikimedia.org |
[production] |
| 09:17 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on ldap-replica1003.wikimedia.org with reason: reboot - T291813 |
[production] |
| 09:17 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.downtime for 0:10:00 on ldap-replica1003.wikimedia.org with reason: reboot - T291813 |
[production] |
| 09:13 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host failoid1002.eqiad.wmnet |
[production] |
| 09:10 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host failoid1002.eqiad.wmnet |
[production] |
| 09:09 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on people1003.eqiad.wmnet with reason: reboot - T291813 |
[production] |
| 09:09 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.downtime for 0:10:00 on people1003.eqiad.wmnet with reason: reboot - T291813 |
[production] |
| 09:07 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on people2002.codfw.wmnet with reason: reboot - T291813 |
[production] |