|
2023-09-19
ยง
|
| 11:48 |
<ayounsi@cumin1001> |
START - Cookbook sre.netbox.update-extras rolling restart_daemons on A:netbox-canary |
[production] |
| 11:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 100%: Repooling after recloning db1128', diff saved to https://phabricator.wikimedia.org/P52530 and previous config saved to /var/cache/conftool/dbconfig/20230919-112156-root.json |
[production] |
| 11:09 |
<Emperor> |
eqiad swift front-end swift package updates T346730 |
[production] |
| 11:06 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 75%: Repooling after recloning db1128', diff saved to https://phabricator.wikimedia.org/P52529 and previous config saved to /var/cache/conftool/dbconfig/20230919-110651-root.json |
[production] |
| 10:51 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 50%: Repooling after recloning db1128', diff saved to https://phabricator.wikimedia.org/P52528 and previous config saved to /var/cache/conftool/dbconfig/20230919-105147-root.json |
[production] |
| 10:38 |
<stevemunene@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-worker1148.eqiad.wmnet with OS bullseye |
[production] |
| 10:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 25%: Repooling after recloning db1128', diff saved to https://phabricator.wikimedia.org/P52527 and previous config saved to /var/cache/conftool/dbconfig/20230919-103642-root.json |
[production] |
| 10:34 |
<Emperor> |
codfw swift front-end swift package updates T346730 |
[production] |
| 10:25 |
<stevemunene@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-worker1147.eqiad.wmnet with OS bullseye |
[production] |
| 10:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 10%: Repooling after recloning db1128', diff saved to https://phabricator.wikimedia.org/P52526 and previous config saved to /var/cache/conftool/dbconfig/20230919-102137-root.json |
[production] |
| 10:15 |
<stevemunene@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-worker1148.eqiad.wmnet with reason: host reimage |
[production] |
| 10:11 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on an-worker1148.eqiad.wmnet with reason: host reimage |
[production] |
| 10:06 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 5%: Repooling after recloning db1128', diff saved to https://phabricator.wikimedia.org/P52525 and previous config saved to /var/cache/conftool/dbconfig/20230919-100632-root.json |
[production] |
| 10:01 |
<stevemunene@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-worker1147.eqiad.wmnet with reason: host reimage |
[production] |
| 09:58 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.reimage for host an-worker1148.eqiad.wmnet with OS bullseye |
[production] |
| 09:56 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on an-worker1147.eqiad.wmnet with reason: host reimage |
[production] |
| 09:51 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 3%: Repooling after recloning db1128', diff saved to https://phabricator.wikimedia.org/P52524 and previous config saved to /var/cache/conftool/dbconfig/20230919-095127-root.json |
[production] |
| 09:48 |
<slyngshede@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host idm2001.wikimedia.org with OS bookworm |
[production] |
| 09:42 |
<stevemunene@cumin1001> |
START - Cookbook sre.hosts.reimage for host an-worker1147.eqiad.wmnet with OS bullseye |
[production] |
| 09:40 |
<btullis@cumin1001> |
START - Cookbook sre.kafka.roll-restart-brokers for Kafka A:kafka-jumbo-eqiad cluster: Roll restart of jvm daemons. |
[production] |
| 09:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1134 (re)pooling @ 1%: Repooling after recloning db1128', diff saved to https://phabricator.wikimedia.org/P52523 and previous config saved to /var/cache/conftool/dbconfig/20230919-093622-root.json |
[production] |
| 09:12 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM ml-staging-etcd2001.codfw.wmnet |
[production] |
| 09:08 |
<elukey@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM ml-staging-etcd2001.codfw.wmnet |
[production] |
| 09:03 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM ml-staging-etcd2002.codfw.wmnet |
[production] |
| 08:59 |
<elukey@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM ml-staging-etcd2002.codfw.wmnet |
[production] |
| 08:47 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM ml-staging-etcd2003.codfw.wmnet |
[production] |
| 08:44 |
<godog> |
bounce benthos@webrequest_live to clear out old metrics |
[production] |
| 08:43 |
<elukey@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM ml-staging-etcd2003.codfw.wmnet |
[production] |
| 08:41 |
<godog> |
remove MediaWiki.*.growthexperiments.taskcount.link_recommendation.* from graphite - T346371 |
[production] |
| 08:39 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.maps.roll-restart-reboot (exit_code=0) rolling restart_daemons on A:maps-replica-eqiad |
[production] |
| 08:36 |
<stevemunene@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-worker1146.eqiad.wmnet with OS bullseye |
[production] |
| 08:34 |
<jmm@cumin2002> |
START - Cookbook sre.maps.roll-restart-reboot rolling restart_daemons on A:maps-replica-eqiad |
[production] |
| 08:30 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.maps.roll-restart-reboot (exit_code=0) rolling restart_daemons on A:maps-replica-codfw |
[production] |
| 08:29 |
<slyngshede@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on idm2001.wikimedia.org with reason: host reimage |
[production] |
| 08:26 |
<brouberol@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
| 08:26 |
<brouberol@deploy1002> |
helmfile [codfw] START helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
| 08:26 |
<slyngshede@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on idm2001.wikimedia.org with reason: host reimage |
[production] |
| 08:26 |
<brouberol> |
redeploying mw-page-content-change-enrich in codfw T336041 |
[production] |
| 08:26 |
<brouberol@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
| 08:25 |
<brouberol@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-page-content-change-enrich: apply |
[production] |
| 08:25 |
<jmm@cumin2002> |
START - Cookbook sre.maps.roll-restart-reboot rolling restart_daemons on A:maps-replica-codfw |
[production] |
| 08:25 |
<brouberol> |
redeploying mw-page-content-change-enrich in eqiad T336041 |
[production] |
| 08:24 |
<brouberol@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/eventstreams-internal: apply |
[production] |
| 08:24 |
<brouberol@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventstreams-internal: apply |
[production] |
| 08:24 |
<brouberol> |
redeploying eventstreams-internal in eqiad T336041 |
[production] |
| 08:23 |
<brouberol@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/eventstreams-internal: apply |
[production] |
| 08:23 |
<brouberol@deploy1002> |
helmfile [codfw] START helmfile.d/services/eventstreams-internal: apply |
[production] |
| 08:23 |
<brouberol> |
redeploying eventstreams-internal in codfw T336041 |
[production] |
| 08:22 |
<brouberol@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/eventgate-analytics-external: apply |
[production] |
| 08:21 |
<brouberol@deploy1002> |
helmfile [codfw] START helmfile.d/services/eventgate-analytics-external: apply |
[production] |