|
2023-08-29
ยง
|
| 14:57 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2181.codfw.wmnet with reason: Maintenance |
[production] |
| 14:57 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2168:3318 (T343718)', diff saved to https://phabricator.wikimedia.org/P51924 and previous config saved to /var/cache/conftool/dbconfig/20230829-145705-ladsgroup.json |
[production] |
| 14:57 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host restbase1028.eqiad.wmnet |
[production] |
| 14:56 |
<jiji@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/cxserver: apply |
[production] |
| 14:56 |
<jiji@deploy1002> |
helmfile [codfw] START helmfile.d/services/cxserver: apply |
[production] |
| 14:56 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw2-d5-eqiad |
[production] |
| 14:56 |
<elukey@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/eventgate-analytics-external: sync |
[production] |
| 14:55 |
<elukey@deploy1002> |
helmfile [codfw] START helmfile.d/services/eventgate-analytics-external: sync |
[production] |
| 14:55 |
<jiji@deploy1002> |
helmfile [staging] DONE helmfile.d/services/cxserver: apply |
[production] |
| 14:55 |
<jiji@deploy1002> |
helmfile [staging] START helmfile.d/services/cxserver: apply |
[production] |
| 14:54 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw2-d5-eqiad |
[production] |
| 14:54 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw1-f4-eqiad |
[production] |
| 14:54 |
<jiji@deploy1002> |
helmfile [staging] DONE helmfile.d/services/cxserver: apply |
[production] |
| 14:54 |
<jiji@deploy1002> |
helmfile [staging] START helmfile.d/services/cxserver: apply |
[production] |
| 14:52 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1214 (T343718)', diff saved to https://phabricator.wikimedia.org/P51923 and previous config saved to /var/cache/conftool/dbconfig/20230829-145242-ladsgroup.json |
[production] |
| 14:51 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-f4-eqiad |
[production] |
| 14:51 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw1-e4-eqiad |
[production] |
| 14:51 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir5002.eqsin.wmnet} and A:ncredir |
[production] |
| 14:51 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir6002.drmrs.wmnet} and A:ncredir |
[production] |
| 14:51 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.reimage for host an-worker1125.eqiad.wmnet with OS bullseye |
[production] |
| 14:50 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.reimage for host an-worker1124.eqiad.wmnet with OS bullseye |
[production] |
| 14:49 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-e4-eqiad |
[production] |
| 14:49 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw1-d5-eqiad |
[production] |
| 14:48 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1028.eqiad.wmnet |
[production] |
| 14:48 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host restbase1021.eqiad.wmnet |
[production] |
| 14:47 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas (exit_code=0) rolling restart_daemons on A:schema-eqiad |
[production] |
| 14:47 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir6002.drmrs.wmnet} and A:ncredir |
[production] |
| 14:47 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-d5-eqiad |
[production] |
| 14:47 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw1-c8-eqiad |
[production] |
| 14:46 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir4002.ulsfo.wmnet} and A:ncredir |
[production] |
| 14:46 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir5002.eqsin.wmnet} and A:ncredir |
[production] |
| 14:46 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas rolling restart_daemons on A:schema-eqiad |
[production] |
| 14:46 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir2002.codfw.wmnet} and A:ncredir |
[production] |
| 14:46 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir1002.eqiad.wmnet} and A:ncredir |
[production] |
| 14:44 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-c8-eqiad |
[production] |
| 14:44 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device cloudsw1-b1-codfw |
[production] |
| 14:44 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device cloudsw1-b1-codfw |
[production] |
| 14:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2140', diff saved to https://phabricator.wikimedia.org/P51922 and previous config saved to /var/cache/conftool/dbconfig/20230829-144349-ladsgroup.json |
[production] |
| 14:42 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir4002.ulsfo.wmnet} and A:ncredir |
[production] |
| 14:42 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir2002.codfw.wmnet} and A:ncredir |
[production] |
| 14:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2168:3318', diff saved to https://phabricator.wikimedia.org/P51921 and previous config saved to /var/cache/conftool/dbconfig/20230829-144159-ladsgroup.json |
[production] |
| 14:41 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir1002.eqiad.wmnet} and A:ncredir |
[production] |
| 14:41 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir2001.codfw.wmnet} and A:ncredir |
[production] |
| 14:41 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-restart-reboot-ncredir (exit_code=0) rolling reboot on P{ncredir6001.drmrs.wmnet} and A:ncredir |
[production] |
| 14:39 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas (exit_code=0) rolling restart_daemons on A:schema-codfw |
[production] |
| 14:38 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1021.eqiad.wmnet |
[production] |
| 14:38 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas rolling restart_daemons on A:schema-codfw |
[production] |
| 14:37 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir2001.codfw.wmnet} and A:ncredir |
[production] |
| 14:37 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-restart-reboot-ncredir rolling reboot on P{ncredir6001.drmrs.wmnet} and A:ncredir |
[production] |
| 14:36 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host kubernetes2025.codfw.wmnet with OS bullseye |
[production] |