|
2023-03-30
§
|
| 08:55 |
<elukey> |
move kafka main clusters to new truststore (PKI+Puppet root CA certs) - T319372 |
[production] |
| 08:54 |
<elukey@cumin1001> |
START - Cookbook sre.kafka.roll-restart-brokers for Kafka A:kafka-main-eqiad cluster: Roll restart of jvm daemons. |
[production] |
| 00:27 |
<pt1979@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['db1207'] |
[production] |
| 00:27 |
<pt1979@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db1207'] |
[production] |
| 00:20 |
<pt1979@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['ms-be1072'] |
[production] |
| 00:20 |
<pt1979@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['ms-be1072'] |
[production] |
| 00:18 |
<pt1979@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['db1207'] |
[production] |
| 00:18 |
<pt1979@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db1207'] |
[production] |
| 00:18 |
<pt1979@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['db1207'] |
[production] |
| 00:18 |
<pt1979@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db1207'] |
[production] |
| 00:13 |
<pt1979@cumin2002> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['db1207'] |
[production] |
| 00:13 |
<pt1979@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['db1207'] |
[production] |
| 00:12 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1225.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 00:11 |
<jclark@cumin1001> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['ms-be1072'] |
[production] |
| 00:10 |
<jclark@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['ms-be1072'] |
[production] |
| 00:10 |
<jclark@cumin1001> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['ms-be1072'] |
[production] |
| 00:09 |
<jclark@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['ms-be1072'] |
[production] |
| 00:08 |
<jclark@cumin1001> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['ms-be1072.eqiad.wmnet'] |
[production] |
| 00:07 |
<jclark@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['ms-be1072.eqiad.wmnet'] |
[production] |
| 00:02 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1225.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
|
2023-03-29
§
|
| 23:59 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1224.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:59 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1223.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:53 |
<dzahn@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on contint2002.wikimedia.org with reason: WIP-known-to-be-debugged-new-host |
[production] |
| 23:52 |
<dzahn@cumin1001> |
START - Cookbook sre.hosts.downtime for 7 days, 0:00:00 on contint2002.wikimedia.org with reason: WIP-known-to-be-debugged-new-host |
[production] |
| 23:51 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1224.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:50 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1223.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:49 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1221.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:49 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1222.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:48 |
<mutante> |
contint2002 - a2dismod mpm_event (ONCE AGAIN this year old issue when applying roles with apache for the first time) - running puppet - now it can actually install PHP 7.3 and start apache T324659 |
[production] |
| 23:48 |
<mutante> |
contint2002 - a2dismod mpm_event (ONCE AGAIN this year old issue when applying roles with apache for the first time) - running puppet - now it can actually install PHP 7.3 and start apache |
[production] |
| 23:23 |
<jclark@cumin1001> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['ms-be1072.eqiad.wmnet'] |
[production] |
| 23:23 |
<jclark@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['ms-be1072.eqiad.wmnet'] |
[production] |
| 23:10 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1222.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:10 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1221.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:09 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1220.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:08 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1219.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:01 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1220.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 23:01 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1219.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:59 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1217.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:57 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1218.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:46 |
<jclark@cumin1001> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['ms-be1072.eqiad.wmnet'] |
[production] |
| 22:37 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1217.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:36 |
<pt1979@cumin2002> |
END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host db1217.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:35 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1218.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:33 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1216.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:28 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1217.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:26 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1215.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:24 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1216.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:23 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host db1214.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |
| 22:18 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.provision for host db1215.mgmt.eqiad.wmnet with reboot policy FORCED |
[production] |