|
2023-11-27
ยง
|
| 15:14 |
<fabfur> |
set `pooled=yes` on cp11.* hosts in eqiad T349244 |
[production] |
| 15:11 |
<jclark@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=93) for host an-worker1160.eqiad.wmnet with OS bullseye |
[production] |
| 15:10 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirtlocal1001.eqiad.wmnet with reason: host reimage |
[production] |
| 15:09 |
<hashar> |
restarting CI Jenkins |
[production] |
| 15:07 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.provision for host elastic2102.mgmt.codfw.wmnet with reboot policy FORCED |
[production] |
| 15:07 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.provision for host elastic2101.mgmt.codfw.wmnet with reboot policy FORCED |
[production] |
| 15:07 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.provision for host elastic2100.mgmt.codfw.wmnet with reboot policy FORCED |
[production] |
| 15:07 |
<fabfur> |
`nfctl select name='cp10.*',service=ats-be set/pooled=inactive` (cdn and ats-be not used anymore on these hosts) T349244 |
[production] |
| 15:07 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.provision for host elastic2099.mgmt.codfw.wmnet with reboot policy FORCED |
[production] |
| 15:07 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.provision for host elastic2098.mgmt.codfw.wmnet with reboot policy FORCED |
[production] |
| 15:06 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.provision for host elastic2097.mgmt.codfw.wmnet with reboot policy FORCED |
[production] |
| 15:06 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirtlocal1001.eqiad.wmnet with reason: host reimage |
[production] |
| 15:02 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2095'] |
[production] |
| 15:01 |
<jclark@cumin1001> |
START - Cookbook sre.hosts.reimage for host an-worker1157.eqiad.wmnet with OS bullseye |
[production] |
| 15:01 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host elastic2096.mgmt.codfw.wmnet with reboot policy FORCED |
[production] |
| 14:58 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2093'] |
[production] |
| 14:57 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2092'] |
[production] |
| 14:57 |
<urbanecm> |
mwmaint2002: `/usr/local/bin/foreachwikiindblist /srv/mediawiki/dblists/growthexperiments.dblist extensions/GrowthExperiments/maintenance/refreshUserImpactData.php --registeredWithin=1year --editedWithin=2week --hasEditsAtLeast=3 --ignoreIfUpdatedWithin=6hour --verbose --use-job-queue` |
[production] |
| 14:56 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2095'] |
[production] |
| 14:56 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['elastic2095'] |
[production] |
| 14:56 |
<urbanecm> |
mwmaint2002: /usr/local/bin/foreachwikiindblist /srv/mediawiki/dblists/growthexperiments.dblist extensions/GrowthExperiments/maintenance/refreshUserImpactData.php --registeredWithin=2week --hasEditsAtLeast=3 --ignoreIfUpdatedWithin=6hour --verbose --use-job-queue |
[production] |
| 14:56 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2095'] |
[production] |
| 14:56 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2094'] |
[production] |
| 14:55 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2094'] |
[production] |
| 14:55 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['elastic2094'] |
[production] |
| 14:55 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2094'] |
[production] |
| 14:55 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.puppet.migrate-host (exit_code=0) for host dns4003.wikimedia.org |
[production] |
| 14:55 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2090'] |
[production] |
| 14:54 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2091'] |
[production] |
| 14:54 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2091'] |
[production] |
| 14:54 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2091'] |
[production] |
| 14:54 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2091'] |
[production] |
| 14:53 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['elastic2091'] |
[production] |
| 14:53 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['elastic2089'] |
[production] |
| 14:53 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2091'] |
[production] |
| 14:53 |
<jclark@cumin1001> |
START - Cookbook sre.hosts.reimage for host an-worker1160.eqiad.wmnet with OS bullseye |
[production] |
| 14:53 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host an-worker1175 |
[production] |
| 14:53 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2093'] |
[production] |
| 14:53 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host an-worker1175 |
[production] |
| 14:53 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host an-worker1161 |
[production] |
| 14:52 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['elastic2093'] |
[production] |
| 14:52 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host an-worker1161 |
[production] |
| 14:52 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host elastic2094.mgmt.codfw.wmnet with reboot policy FORCED |
[production] |
| 14:52 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host an-worker1159 |
[production] |
| 14:52 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host an-worker1159 |
[production] |
| 14:52 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host an-worker1158 |
[production] |
| 14:52 |
<jhancock@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['elastic2093'] |
[production] |
| 14:52 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host an-worker1158 |
[production] |
| 14:52 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host an-worker1157 |
[production] |
| 14:52 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host an-worker1157 |
[production] |