|
2023-08-23
ยง
|
| 12:26 |
<gmodena@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
| 12:26 |
<gmodena@deploy1002> |
helmfile [codfw] START helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
| 12:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2181 (T344589)', diff saved to https://phabricator.wikimedia.org/P51102 and previous config saved to /var/cache/conftool/dbconfig/20230823-122440-ladsgroup.json |
[production] |
| 12:19 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-reboot rolling reboot on A:cp-text_eqiad and A:cp |
[production] |
| 12:19 |
<fabfur@cumin1001> |
START - Cookbook sre.cdn.roll-reboot rolling reboot on A:cp-upload_eqiad and A:cp |
[production] |
| 12:17 |
<klausman@cumin1001> |
END (PASS) - Cookbook sre.k8s.reboot-nodes (exit_code=0) rolling reboot on A:ml-serve-worker-codfw |
[production] |
| 12:14 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas (exit_code=0) rolling reboot on A:schema-codfw |
[production] |
| 12:12 |
<btullis@cumin1001> |
START - Cookbook sre.kafka.reboot-workers for Kafka test-eqiad cluster: Reboot kafka nodes |
[production] |
| 12:11 |
<gmodena@deploy1002> |
helmfile [staging] DONE helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
| 12:11 |
<gmodena@deploy1002> |
helmfile [staging] START helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
| 12:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2181', diff saved to https://phabricator.wikimedia.org/P51101 and previous config saved to /var/cache/conftool/dbconfig/20230823-120933-ladsgroup.json |
[production] |
| 12:03 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas rolling reboot on A:schema-codfw |
[production] |
| 12:00 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-upload_drmrs and A:cp |
[production] |
| 11:59 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-text_drmrs and A:cp |
[production] |
| 11:54 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host debmonitor1002.eqiad.wmnet |
[production] |
| 11:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2181', diff saved to https://phabricator.wikimedia.org/P51100 and previous config saved to /var/cache/conftool/dbconfig/20230823-115427-ladsgroup.json |
[production] |
| 11:51 |
<btullis@cumin1001> |
END (PASS) - Cookbook sre.kafka.reboot-workers (exit_code=0) for Kafka jumbo-eqiad cluster: Reboot kafka nodes |
[production] |
| 11:51 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host debmonitor1002.eqiad.wmnet |
[production] |
| 11:41 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-reboot (exit_code=0) rolling reboot on A:cp-text_ulsfo and A:cp |
[production] |
| 11:39 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2181 (T344589)', diff saved to https://phabricator.wikimedia.org/P51099 and previous config saved to /var/cache/conftool/dbconfig/20230823-113921-ladsgroup.json |
[production] |
| 11:37 |
<btullis@cumin1001> |
START - Cookbook sre.druid.reboot-workers for Druid public cluster: Reboot Druid nodes |
[production] |
| 11:36 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ldap.roll-restart-reboot-replica (exit_code=0) rolling reboot on A:ldap-replicas-eqiad |
[production] |
| 11:35 |
<vgutierrez@cumin1001> |
START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-upload_drmrs and A:cp |
[production] |
| 11:35 |
<vgutierrez@cumin1001> |
START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-text_drmrs and A:cp |
[production] |
| 11:33 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2181 (T344589)', diff saved to https://phabricator.wikimedia.org/P51098 and previous config saved to /var/cache/conftool/dbconfig/20230823-113310-ladsgroup.json |
[production] |
| 11:33 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2181.codfw.wmnet with reason: Maintenance |
[production] |
| 11:32 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2181.codfw.wmnet with reason: Maintenance |
[production] |
| 11:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2167:3318 (T344589)', diff saved to https://phabricator.wikimedia.org/P51097 and previous config saved to /var/cache/conftool/dbconfig/20230823-113244-ladsgroup.json |
[production] |
| 11:32 |
<btullis@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host an-presto1002.eqiad.wmnet |
[production] |
| 11:31 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-text_eqsin and A:cp |
[production] |
| 11:30 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.cdn.roll-upgrade-haproxy (exit_code=0) rolling upgrade of HAProxy on A:cp-upload_eqsin and A:cp |
[production] |
| 11:28 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host debmonitor2002.codfw.wmnet |
[production] |
| 11:28 |
<jmm@cumin2002> |
START - Cookbook sre.ldap.roll-restart-reboot-replica rolling reboot on A:ldap-replicas-eqiad |
[production] |
| 11:25 |
<btullis@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host an-presto1002.eqiad.wmnet |
[production] |
| 11:25 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host atlas2001.wikimedia.org |
[production] |
| 11:25 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM atlas2001.wikimedia.org - ayounsi@cumin1001" |
[production] |
| 11:24 |
<ayounsi@cumin1001> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM atlas2001.wikimedia.org - ayounsi@cumin1001" |
[production] |
| 11:24 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host debmonitor2002.codfw.wmnet |
[production] |
| 11:23 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.ldap.roll-restart-reboot-replica (exit_code=0) rolling reboot on A:ldap-replicas-codfw |
[production] |
| 11:21 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) atlas2001.wikimedia.org on all recursors |
[production] |
| 11:21 |
<ayounsi@cumin1001> |
START - Cookbook sre.dns.wipe-cache atlas2001.wikimedia.org on all recursors |
[production] |
| 11:21 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 11:21 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM atlas2001.wikimedia.org - ayounsi@cumin1001" |
[production] |
| 11:19 |
<ayounsi@cumin1001> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Add records for VM atlas2001.wikimedia.org - ayounsi@cumin1001" |
[production] |
| 11:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2167:3318', diff saved to https://phabricator.wikimedia.org/P51096 and previous config saved to /var/cache/conftool/dbconfig/20230823-111737-ladsgroup.json |
[production] |
| 11:17 |
<ayounsi@cumin1001> |
START - Cookbook sre.dns.netbox |
[production] |
| 11:17 |
<ayounsi@cumin1001> |
START - Cookbook sre.ganeti.makevm for new host atlas2001.wikimedia.org |
[production] |
| 11:15 |
<jmm@cumin2002> |
START - Cookbook sre.ldap.roll-restart-reboot-replica rolling reboot on A:ldap-replicas-codfw |
[production] |
| 11:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2177 (T343718)', diff saved to https://phabricator.wikimedia.org/P51095 and previous config saved to /var/cache/conftool/dbconfig/20230823-111500-ladsgroup.json |
[production] |
| 11:02 |
<vgutierrez@cumin1001> |
START - Cookbook sre.cdn.roll-upgrade-haproxy rolling upgrade of HAProxy on A:cp-upload_eqsin and A:cp |
[production] |