|
2026-11-14
§
|
| 15:44 |
<fabfur@cumin1004> |
conftool action : tet; selector: cluster=cache |
[production] |
| 14:39 |
<sukhe> |
sudo cumin -b1 -s30 "A:dnsbox and (A:eqiad or A:codfw)" "birdc restart bfd1" |
[production] |
| 14:34 |
<sukhe> |
sudo cumin -b1 -s5 "A:wikidough and (A:eqiad or A:codfw)" "birdc restart bfd1": T434806 |
[production] |
| 13:16 |
<brett@cumin2003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on cp3074.esams.wmnet with reason: troubleshooting cmos failure |
[production] |
| 12:08 |
<sukhe@cumin1004> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3 days, 0:00:00 on cp3074.esams.wmnet with reason: host is depooled for HW failure, depooling |
[production] |
| 12:00 |
<root@deploy2003> |
helmfile [codfw] DONE helmfile.d/services/shellbox-timeline: sync |
[production] |
| 12:00 |
<root@deploy2003> |
helmfile [codfw] START helmfile.d/services/shellbox-timeline: sync |
[production] |
| 12:00 |
<root@deploy2003> |
helmfile [staging] DONE helmfile.d/services/shellbox: sync |
[production] |
| 12:00 |
<root@deploy2003> |
helmfile [staging] START helmfile.d/services/shellbox: sync |
[production] |
| 11:04 |
<sukhe@cumin1004> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-hcaptcha-proxy (exit_code=0) rolling reboot on A:hcaptcha-proxy and A:hcaptcha-proxy |
[production] |
| 11:01 |
<jmm@cumin2003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host bast5005.wikimedia.org |
[production] |
| 11:01 |
<sukhe@cumin1004> |
END (PASS) - Cookbook sre.dns.roll-restart-reboot-wikimedia-dns (exit_code=0) rolling reboot on A:wikidough |
[production] |
| 11:01 |
<jmm@cumin2003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host bast4006.wikimedia.org |
[production] |
| 10:55 |
<jmm@cumin2003> |
START - Cookbook sre.hosts.reboot-single for host bast5005.wikimedia.org |
[production] |
| 10:55 |
<jmm@cumin2003> |
START - Cookbook sre.hosts.reboot-single for host bast4006.wikimedia.org |
[production] |
| 10:52 |
<jmm@cumin2003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host bast2003.wikimedia.org |
[production] |
| 10:44 |
<jmm@cumin2003> |
START - Cookbook sre.hosts.reboot-single for host bast2003.wikimedia.org |
[production] |
| 10:43 |
<jmm@cumin2003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host bast1004.wikimedia.org |
[production] |
| 10:39 |
<sukhe@cumin1004> |
END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-urldownloaders (exit_code=0) rolling reboot on A:url-downloader |
[production] |
| 10:37 |
<jmm@cumin2003> |
START - Cookbook sre.hosts.reboot-single for host bast1004.wikimedia.org |
[production] |
| 10:01 |
<sukhe@cumin1004> |
START - Cookbook sre.misc-clusters.roll-restart-reboot-urldownloaders rolling reboot on A:url-downloader |
[production] |
| 09:58 |
<sukhe@cumin1004> |
START - Cookbook sre.cdn.roll-restart-reboot-hcaptcha-proxy rolling reboot on A:hcaptcha-proxy and A:hcaptcha-proxy |
[production] |
| 09:58 |
<sukhe@cumin1004> |
START - Cookbook sre.dns.roll-restart-reboot-wikimedia-dns rolling reboot on A:wikidough |
[production] |
| 05:23 |
<vgutierrez@cumin1004> |
END (PASS) - Cookbook sre.dns.roll-reboot (exit_code=0) rolling reboot on A:dnsbox |
[production] |
| 05:23 |
<vgutierrez@cumin1004> |
cookbooks.sre.dns.roll-reboot finished rebooting dns7002.wikimedia.org |
[production] |
| 05:12 |
<vgutierrez@cumin1004> |
cookbooks.sre.dns.roll-reboot begin reboot of dns7002.wikimedia.org |
[production] |
| 04:57 |
<vgutierrez@cumin1004> |
cookbooks.sre.dns.roll-reboot finished rebooting dns7001.wikimedia.org |
[production] |
| 04:42 |
<vgutierrez@cumin1004> |
cookbooks.sre.dns.roll-reboot begin reboot of dns7001.wikimedia.org |
[production] |
| 04:36 |
<vgutierrez@cumin1004> |
END (PASS) - Cookbook sre.cdn.roll-reboot (exit_code=0) rolling reboot on P{cp3075.*} and A:cp |
[production] |
| 04:36 |
<vgutierrez@cumin1004> |
cookbooks.sre.cdn.roll-reboot finished rebooting cp3075.esams.wmnet |
[production] |
| 04:27 |
<vgutierrez@cumin1004> |
cookbooks.sre.dns.roll-reboot finished rebooting dns6002.wikimedia.org |
[production] |
| 04:25 |
<vgutierrez@cumin1004> |
START - Cookbook sre.cdn.roll-reboot rolling reboot on P{cp3075.*} and A:cp |
[production] |
| 04:21 |
<vgutierrez@cumin1004> |
END (ERROR) - Cookbook sre.cdn.roll-reboot (exit_code=97) rolling reboot on A:cp-text_esams |
[production] |
| 04:20 |
<vgutierrez@cumin1004> |
END (ERROR) - Cookbook sre.cdn.roll-reboot (exit_code=97) rolling reboot on A:cp-text_eqiad |
[production] |
| 04:20 |
<vgutierrez@cumin1004> |
START - Cookbook sre.cdn.roll-reboot rolling reboot on A:cp-text_eqiad |
[production] |
| 04:06 |
<vgutierrez@cumin1004> |
cookbooks.sre.dns.roll-reboot begin reboot of dns6002.wikimedia.org |
[production] |
| 03:51 |
<vgutierrez@cumin1004> |
cookbooks.sre.dns.roll-reboot finished rebooting dns6001.wikimedia.org |
[production] |
| 03:37 |
<vgutierrez@cumin1004> |
cookbooks.sre.dns.roll-reboot begin reboot of dns6001.wikimedia.org |
[production] |
| 03:26 |
<vgutierrez@cumin1004> |
END (PASS) - Cookbook sre.cdn.roll-reboot (exit_code=0) rolling reboot on A:cp-text_eqsin |
[production] |
| 03:26 |
<vgutierrez@cumin1004> |
cookbooks.sre.cdn.roll-reboot finished rebooting cp5026.eqsin.wmnet |
[production] |
| 03:22 |
<vgutierrez@cumin1004> |
cookbooks.sre.dns.roll-reboot finished rebooting dns5004.wikimedia.org |
[production] |
| 03:22 |
<vgutierrez@cumin1004> |
END (PASS) - Cookbook sre.cdn.roll-reboot (exit_code=0) rolling reboot on A:cp-text_codfw |
[production] |
| 03:22 |
<vgutierrez@cumin1004> |
cookbooks.sre.cdn.roll-reboot finished rebooting cp2059.codfw.wmnet |
[production] |
| 03:11 |
<vgutierrez@cumin1004> |
END (PASS) - Cookbook sre.cdn.roll-reboot (exit_code=0) rolling reboot on A:cp-text_eqiad |
[production] |