|
2022-09-01
ยง
|
| 18:50 |
<pt1979@cumin2002> |
START - Cookbook sre.network.configure-switch-interfaces for host db1198 |
[production] |
| 18:50 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host db1197 |
[production] |
| 18:49 |
<pt1979@cumin2002> |
START - Cookbook sre.network.configure-switch-interfaces for host db1197 |
[production] |
| 18:49 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host db1196 |
[production] |
| 18:49 |
<pt1979@cumin2002> |
START - Cookbook sre.network.configure-switch-interfaces for host db1196 |
[production] |
| 18:48 |
<pt1979@cumin1001> |
START - Cookbook sre.dns.netbox |
[production] |
| 18:48 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 18:48 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 18:42 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 18:42 |
<dduvall@deploy1002> |
rebuilt and synchronized wikiversions files: all wikis to 1.39.0-wmf.27 refs T314188 |
[production] |
| 17:34 |
<bd808@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/developer-portal: apply |
[production] |
| 17:33 |
<bd808@deploy1002> |
helmfile [eqiad] START helmfile.d/services/developer-portal: apply |
[production] |
| 17:33 |
<bd808@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/developer-portal: apply |
[production] |
| 17:32 |
<bd808@deploy1002> |
helmfile [codfw] START helmfile.d/services/developer-portal: apply |
[production] |
| 17:32 |
<bd808@deploy1002> |
helmfile [staging] DONE helmfile.d/services/developer-portal: apply |
[production] |
| 17:31 |
<bd808@deploy1002> |
helmfile [staging] START helmfile.d/services/developer-portal: apply |
[production] |
| 17:26 |
<herron> |
restarted rsyslog on centrallog2002 |
[production] |
| 16:29 |
<topranks> |
Brining Lumen Tranport CCT 442550294 (cr1-codfw to cr4-ulsfo) back into service following successful hot-cut to lower-latency path with carrier |
[production] |
| 16:17 |
<hnowlan@puppetmaster1001> |
conftool action : set/weight=10; selector: name=restbase103[1-3].eqiad.wmnet |
[production] |
| 15:55 |
<hnowlan@puppetmaster1001> |
conftool action : set/pooled=yes; selector: name=restbase103[1-3].eqiad.wmnet |
[production] |
| 15:35 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 15:35 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 15:34 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 15:34 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 15:21 |
<moritzm> |
installing usb.ids update from Bullseye 11.4 point release |
[production] |
| 15:19 |
<moritzm> |
updating docker.io on ml-serve* to bugfix release from Bullseye 11.4 point release |
[production] |
| 14:54 |
<topranks> |
Draining traffic from Lumen Tranport CCT 442550294 (cr1-codfw to cr4-ulsfo) ahead of hot-cut to lower-latency path with carrier |
[production] |
| 14:29 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetboard1002.eqiad.wmnet |
[production] |
| 14:25 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host puppetboard1002.eqiad.wmnet |
[production] |
| 14:07 |
<moritzm> |
installing net-snmp security updates on Buster |
[production] |
| 14:01 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host netboxdb1002.eqiad.wmnet |
[production] |
| 14:01 |
<marostegui> |
test T316744 |
[production] |
| 14:01 |
<marostegui> |
test T316744 |
[production] |
| 14:00 |
<marostegui> |
Failover m5 from db1107 to db1183 - T316744 |
[production] |
| 13:57 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host netboxdb1002.eqiad.wmnet |
[production] |
| 13:56 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host netboxdb2002.codfw.wmnet |
[production] |
| 13:53 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host netboxdb2002.codfw.wmnet |
[production] |
| 13:52 |
<jmm@cumin2002> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=1) for host netbox1002.eqiad.wmnet |
[production] |
| 13:43 |
<moritzm> |
rebooting netbox1002 (running netbox.wikimedia.org) |
[production] |
| 13:43 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host netbox1002.eqiad.wmnet |
[production] |
| 13:41 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host netbox2002.codfw.wmnet |
[production] |
| 13:37 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host netbox2002.codfw.wmnet |
[production] |
| 13:32 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on db[2135,2160].codfw.wmnet,db[1107,1117,1183].eqiad.wmnet with reason: switchover m5 T316744 |
[production] |
| 13:31 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on db[2135,2160].codfw.wmnet,db[1107,1117,1183].eqiad.wmnet with reason: switchover m5 T316744 |
[production] |
| 13:19 |
<jayme@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 13:19 |
<jayme@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 13:19 |
<jayme@deploy1002> |
helmfile [codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 13:19 |
<jayme@deploy1002> |
helmfile [codfw] START helmfile.d/admin 'apply'. |
[production] |
| 13:18 |
<jayme@deploy1002> |
helmfile [staging-eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 13:18 |
<jayme@deploy1002> |
helmfile [staging-eqiad] START helmfile.d/admin 'apply'. |
[production] |