|
2023-08-29
ยง
|
| 14:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2140', diff saved to https://phabricator.wikimedia.org/P51922 and previous config saved to /var/cache/conftool/dbconfig/20230829-144349-ladsgroup.json |
[production] |
| 14:42 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir4002.ulsfo.wmnet} and A:ncredir |
[production] |
| 14:42 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir2002.codfw.wmnet} and A:ncredir |
[production] |
| 14:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2168:3318', diff saved to https://phabricator.wikimedia.org/P51921 and previous config saved to /var/cache/conftool/dbconfig/20230829-144159-ladsgroup.json |
[production] |
| 14:41 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir1002.eqiad.wmnet} and A:ncredir |
[production] |
| 14:41 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir2001.codfw.wmnet} and A:ncredir |
[production] |
| 14:41 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir6001.drmrs.wmnet} and A:ncredir |
[production] |
| 14:39 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas (exit_code=0) rolling restart_daemons on A:schema-codfw |
[production] |
| 14:38 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1021.eqiad.wmnet |
[production] |
| 14:38 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas rolling restart_daemons on A:schema-codfw |
[production] |
| 14:37 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir2001.codfw.wmnet} and A:ncredir |
[production] |
| 14:37 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir6001.drmrs.wmnet} and A:ncredir |
[production] |
| 14:36 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host kubernetes2025.codfw.wmnet with OS bullseye |
[production] |
| 14:35 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir5001.eqsin.wmnet} and A:ncredir |
[production] |
| 14:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1214 (T343718)', diff saved to https://phabricator.wikimedia.org/P51920 and previous config saved to /var/cache/conftool/dbconfig/20230829-143434-ladsgroup.json |
[production] |
| 14:34 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1214.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:34 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1214.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1211 (T343718)', diff saved to https://phabricator.wikimedia.org/P51919 and previous config saved to /var/cache/conftool/dbconfig/20230829-143413-ladsgroup.json |
[production] |
| 14:32 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device asw2-22-ulsfo |
[production] |
| 14:31 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir4001.ulsfo.wmnet} and A:ncredir |
[production] |
| 14:31 |
<jmm@cumin2002> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1018.eqiad.wmnet |
[production] |
| 14:30 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device asw2-22-ulsfo |
[production] |
| 14:29 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cr4-ulsfo |
[production] |
| 14:28 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir5001.eqsin.wmnet} and A:ncredir |
[production] |
| 14:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2140 (T343718)', diff saved to https://phabricator.wikimedia.org/P51918 and previous config saved to /var/cache/conftool/dbconfig/20230829-142843-ladsgroup.json |
[production] |
| 14:28 |
<fabfur@cumin1001> |
END (ERROR) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=97) rolling reboot on P{ncredir5001.*} and A:ncredir |
[production] |
| 14:28 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir5001.*} and A:ncredir |
[production] |
| 14:28 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host restbase1018.eqiad.wmnet |
[production] |
| 14:27 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir4001.ulsfo.wmnet} and A:ncredir |
[production] |
| 14:27 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir1001.eqiad.wmnet} and A:ncredir |
[production] |
| 14:26 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2168:3318', diff saved to https://phabricator.wikimedia.org/P51917 and previous config saved to /var/cache/conftool/dbconfig/20230829-142653-ladsgroup.json |
[production] |
| 14:25 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1017.eqiad.wmnet |
[production] |
| 14:25 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cr4-ulsfo |
[production] |
| 14:25 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1017.eqiad.wmnet |
[production] |
| 14:22 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir1001.eqiad.wmnet} and A:ncredir |
[production] |
| 14:21 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host ganeti1017.eqiad.wmnet |
[production] |
| 14:19 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1018.eqiad.wmnet |
[production] |
| 14:19 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1211', diff saved to https://phabricator.wikimedia.org/P51916 and previous config saved to /var/cache/conftool/dbconfig/20230829-141907-ladsgroup.json |
[production] |
| 14:14 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host restbase1017.eqiad.wmnet |
[production] |
| 14:11 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2168:3318 (T343718)', diff saved to https://phabricator.wikimedia.org/P51915 and previous config saved to /var/cache/conftool/dbconfig/20230829-141147-ladsgroup.json |
[production] |
| 14:08 |
<hnowlan@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/geo-analytics: apply |
[production] |
| 14:08 |
<hnowlan@deploy1002> |
helmfile [eqiad] START helmfile.d/services/geo-analytics: apply |
[production] |
| 14:08 |
<fabfur> |
start rebooting ncredir hosts for T344587 |
[production] |
| 14:07 |
<elukey@deploy1002> |
helmfile [staging] DONE helmfile.d/services/eventgate-analytics-external: sync |
[production] |
| 14:07 |
<elukey@deploy1002> |
helmfile [staging] START helmfile.d/services/eventgate-analytics-external: sync |
[production] |
| 14:06 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1017.eqiad.wmnet |
[production] |
| 14:06 |
<hnowlan@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/geo-analytics: apply |
[production] |
| 14:06 |
<eevans@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=99) for host restbase1017.eqiad.wmnet |
[production] |
| 14:06 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1017.eqiad.wmnet |
[production] |
| 14:06 |
<hnowlan@deploy1002> |
helmfile [codfw] START helmfile.d/services/geo-analytics: apply |
[production] |