|
2023-08-29
ยง
|
| 14:52 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1214 (T343718)', diff saved to https://phabricator.wikimedia.org/P51923 and previous config saved to /var/cache/conftool/dbconfig/20230829-145242-ladsgroup.json |
[production] |
| 14:51 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-f4-eqiad |
[production] |
| 14:51 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw1-e4-eqiad |
[production] |
| 14:51 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir5002.eqsin.wmnet} and A:ncredir |
[production] |
| 14:51 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir6002.drmrs.wmnet} and A:ncredir |
[production] |
| 14:51 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.reimage for host an-worker1125.eqiad.wmnet with OS bullseye |
[production] |
| 14:50 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.reimage for host an-worker1124.eqiad.wmnet with OS bullseye |
[production] |
| 14:49 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-e4-eqiad |
[production] |
| 14:49 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw1-d5-eqiad |
[production] |
| 14:48 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1028.eqiad.wmnet |
[production] |
| 14:48 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host restbase1021.eqiad.wmnet |
[production] |
| 14:47 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas (exit_code=0) rolling restart_daemons on A:schema-eqiad |
[production] |
| 14:47 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir6002.drmrs.wmnet} and A:ncredir |
[production] |
| 14:47 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-d5-eqiad |
[production] |
| 14:47 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw1-c8-eqiad |
[production] |
| 14:46 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir4002.ulsfo.wmnet} and A:ncredir |
[production] |
| 14:46 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir5002.eqsin.wmnet} and A:ncredir |
[production] |
| 14:46 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas rolling restart_daemons on A:schema-eqiad |
[production] |
| 14:46 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir2002.codfw.wmnet} and A:ncredir |
[production] |
| 14:46 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir1002.eqiad.wmnet} and A:ncredir |
[production] |
| 14:44 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-c8-eqiad |
[production] |
| 14:44 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw1-b1-codfw |
[production] |
| 14:44 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-b1-codfw |
[production] |
| 14:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2140', diff saved to https://phabricator.wikimedia.org/P51922 and previous config saved to /var/cache/conftool/dbconfig/20230829-144349-ladsgroup.json |
[production] |
| 14:42 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir4002.ulsfo.wmnet} and A:ncredir |
[production] |
| 14:42 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir2002.codfw.wmnet} and A:ncredir |
[production] |
| 14:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2168:3318', diff saved to https://phabricator.wikimedia.org/P51921 and previous config saved to /var/cache/conftool/dbconfig/20230829-144159-ladsgroup.json |
[production] |
| 14:41 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir1002.eqiad.wmnet} and A:ncredir |
[production] |
| 14:41 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir2001.codfw.wmnet} and A:ncredir |
[production] |
| 14:41 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir6001.drmrs.wmnet} and A:ncredir |
[production] |
| 14:39 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas (exit_code=0) rolling restart_daemons on A:schema-codfw |
[production] |
| 14:38 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1021.eqiad.wmnet |
[production] |
| 14:38 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas rolling restart_daemons on A:schema-codfw |
[production] |
| 14:37 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir2001.codfw.wmnet} and A:ncredir |
[production] |
| 14:37 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir6001.drmrs.wmnet} and A:ncredir |
[production] |
| 14:36 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host kubernetes2025.codfw.wmnet with OS bullseye |
[production] |
| 14:35 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir5001.eqsin.wmnet} and A:ncredir |
[production] |
| 14:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1214 (T343718)', diff saved to https://phabricator.wikimedia.org/P51920 and previous config saved to /var/cache/conftool/dbconfig/20230829-143434-ladsgroup.json |
[production] |
| 14:34 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1214.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:34 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1214.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1211 (T343718)', diff saved to https://phabricator.wikimedia.org/P51919 and previous config saved to /var/cache/conftool/dbconfig/20230829-143413-ladsgroup.json |
[production] |
| 14:32 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device asw2-22-ulsfo |
[production] |
| 14:31 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir4001.ulsfo.wmnet} and A:ncredir |
[production] |
| 14:31 |
<jmm@cumin2002> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1018.eqiad.wmnet |
[production] |
| 14:30 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device asw2-22-ulsfo |
[production] |
| 14:29 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cr4-ulsfo |
[production] |
| 14:28 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir5001.eqsin.wmnet} and A:ncredir |
[production] |
| 14:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2140 (T343718)', diff saved to https://phabricator.wikimedia.org/P51918 and previous config saved to /var/cache/conftool/dbconfig/20230829-142843-ladsgroup.json |
[production] |
| 14:28 |
<fabfur@cumin1001> |
END (ERROR) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=97) rolling reboot on P{ncredir5001.*} and A:ncredir |
[production] |
| 14:28 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir5001.*} and A:ncredir |
[production] |