|
2023-06-21
ยง
|
| 23:09 |
<mutante> |
created temporary test VM phab-test1001.eqiad.wmnet which we need for a one-time test for T335080 - it will soon be destroyed again |
[production] |
| 23:08 |
<dzahn@cumin1001> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM phab-test1001.eqiad.wmnet - dzahn@cumin1001" |
[production] |
| 23:08 |
<dzahn@cumin1001> |
END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) phab-test1001.eqiad.wmnet on all recursors |
[production] |
| 23:07 |
<dzahn@cumin1001> |
START - Cookbook sre.dns.wipe-cache phab-test1001.eqiad.wmnet on all recursors |
[production] |
| 23:07 |
<dzahn@cumin1001> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 23:07 |
<dzahn@cumin1001> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM phab-test1001.eqiad.wmnet - dzahn@cumin1001" |
[production] |
| 23:07 |
<dzahn@cumin1001> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM phab-test1001.eqiad.wmnet - dzahn@cumin1001" |
[production] |
| 23:02 |
<dzahn@cumin1001> |
START - Cookbook sre.dns.netbox |
[production] |
| 23:02 |
<dzahn@cumin1001> |
START - Cookbook sre.ganeti.makevm for new host phab-test1001.eqiad.wmnet |
[production] |
| 23:01 |
<jclark@cumin1001> |
START - Cookbook sre.hosts.reimage for host dbproxy1023.eqiad.wmnet with OS bullseye |
[production] |
| 23:00 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host dbproxy1023 |
[production] |
| 22:58 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host dbproxy1023 |
[production] |
| 22:55 |
<bking@cumin1001> |
END (FAIL) - Cookbook sre.wdqs.data-transfer (exit_code=99) |
[production] |
| 22:54 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 22:54 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt dbproxy1023 - jclark@cumin1001" |
[production] |
| 22:53 |
<jclark@cumin1001> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt dbproxy1023 - jclark@cumin1001" |
[production] |
| 22:52 |
<brett@cumin2002> |
END (PASS) - Cookbook sre.cdn.roll-reboot (exit_code=0) rolling reboot on P{cp1078.eqiad.wmnet,cp1080.eqiad.wmnet,cp1082.eqiad.wmnet,cp1084.eqiad.wmnet,cp1086.eqiad.wmnet,cp1088.eqiad.wmnet,cp1090.eqiad.wmnet} and A:cp |
[production] |
| 22:51 |
<jclark@cumin1001> |
START - Cookbook sre.dns.netbox |
[production] |
| 22:48 |
<brett@cumin2002> |
END (PASS) - Cookbook sre.cdn.roll-reboot (exit_code=0) rolling reboot on P{cp1077.eqiad.wmnet,cp1079.eqiad.wmnet,cp1081.eqiad.wmnet,cp1083.eqiad.wmnet,cp1085.eqiad.wmnet,cp1087.eqiad.wmnet,cp1089.eqiad.wmnet} and A:cp |
[production] |
| 22:39 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host dbproxy1023 |
[production] |
| 22:39 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host dbproxy1023 |
[production] |
| 22:39 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host dbproxy1023 |
[production] |
| 22:39 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host dbproxy1023 |
[production] |
| 22:38 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host dbproxy1023 |
[production] |
| 22:38 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host dbproxy1023 |
[production] |
| 22:38 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host dbproxy1023 |
[production] |
| 22:38 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host dbproxy1023 |
[production] |
| 22:38 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host dbproxy1023 |
[production] |
| 22:38 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host dbproxy1023 |
[production] |
| 22:36 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 22:36 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt dbproxy1023 - jclark@cumin1001" |
[production] |
| 22:35 |
<jclark@cumin1001> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt dbproxy1023 - jclark@cumin1001" |
[production] |
| 22:33 |
<jclark@cumin1001> |
START - Cookbook sre.dns.netbox |
[production] |
| 22:33 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host dbproxy1023 |
[production] |
| 22:33 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host dbproxy1023 |
[production] |
| 22:33 |
<jclark@cumin1001> |
END (PASS) - Cookbook sre.network.configure-switch-interfaces (exit_code=0) for host dbproxy1023 |
[production] |
| 22:33 |
<jclark@cumin1001> |
START - Cookbook sre.network.configure-switch-interfaces for host dbproxy1023 |
[production] |
| 22:26 |
<dzahn@cumin1001> |
END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts gerrit1001.wikimedia.org |
[production] |
| 22:26 |
<dzahn@cumin1001> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 22:26 |
<dzahn@cumin1001> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: gerrit1001.wikimedia.org decommissioned, removing all IPs except the asset tag one - dzahn@cumin1001" |
[production] |
| 22:25 |
<dzahn@cumin1001> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: gerrit1001.wikimedia.org decommissioned, removing all IPs except the asset tag one - dzahn@cumin1001" |
[production] |
| 22:22 |
<dzahn@cumin1001> |
START - Cookbook sre.dns.netbox |
[production] |
| 22:19 |
<eevans@cumin2002> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host sessionstore2001.codfw.wmnet with OS bullseye |
[production] |
| 22:16 |
<mutante> |
destroying previous production gerrit server gerrit1001 - T336427 |
[production] |
| 22:15 |
<dzahn@cumin1001> |
START - Cookbook sre.hosts.decommission for hosts gerrit1001.wikimedia.org |
[production] |
| 22:10 |
<mutante> |
rsyncing data from cobalt.wikimedia.org (:p) from gerrit1001 to gerrit1003, /srv/gerrit/cobalt/ |
[production] |
| 21:30 |
<eevans@cumin2002> |
START - Cookbook sre.hosts.reimage for host sessionstore2001.codfw.wmnet with OS bullseye |
[production] |
| 21:28 |
<eevans@cumin2002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['sessionstore2001.codfw.wmnet'] |
[production] |
| 21:24 |
<eevans@cumin2002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['sessionstore2001.codfw.wmnet'] |
[production] |
| 21:23 |
<eevans@cumin2002> |
END (ERROR) - Cookbook sre.hardware.upgrade-firmware (exit_code=97) upgrade firmware for hosts ['sessionstore2001.codfw.wmnet'] |
[production] |