|
2024-06-24
§
|
| 22:24 |
<cwhite@cumin2002> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: logstash2001.codfw.wmnet decommissioned, removing all IPs except the asset tag one - cwhite@cumin2002" |
[production] |
| 22:18 |
<cwhite@cumin2002> |
START - Cookbook sre.dns.netbox |
[production] |
| 22:11 |
<cwhite@cumin2002> |
START - Cookbook sre.hosts.decommission for hosts logstash2001.codfw.wmnet |
[production] |
| 21:34 |
<inflatador> |
bking@alerts1001 uninstall deb pkg `ripgrep` T368107 |
[production] |
| 21:03 |
<btullis@cumin1002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host an-redacteddb1001.eqiad.wmnet |
[production] |
| 21:00 |
<eevans@cumin1002> |
END (PASS) - Cookbook sre.cassandra.roll-restart (exit_code=0) for nodes matching A:restbase-codfw: Apply Cassandra upgrade to 4.1.5 — T354970 - eevans@cumin1002 |
[production] |
| 20:56 |
<andrewbogott> |
rebooting tools-k8s-worker-nfs-36; it has lots of stuck processes which somehow didn't get unstuck when we did the post-nfs-migration reboots. |
[tools] |
| 20:53 |
<btullis@cumin1002> |
START - Cookbook sre.hosts.reboot-single for host an-redacteddb1001.eqiad.wmnet |
[production] |
| 20:36 |
<inflatador> |
bking@alert1001 install `ripgrep` deb pkg T368107 |
[production] |
| 20:22 |
<ladsgroup@deploy1002> |
Synchronized php-1.43.0-wmf.10/includes/libs/rdbms/loadbalancer/LoadBalancer.php: (no justification provided) (duration: 11m 04s) |
[production] |
| 20:21 |
<mutante> |
snapsho1017 - systemctl mask commonsrdf-dump ; systemctl mask commonsjson-dump T368098 |
[production] |
| 20:18 |
<taavi> |
taavi@snapshot1017 ~ $ sudo systemctl stop commons*.service |
[production] |
| 20:09 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.openstack.migrate_project_to_ovs (exit_code=1) |
[metricsinfra] |
| 20:01 |
<andrew@cumin1002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cloudvirt1056.eqiad.wmnet with OS bookworm |
[production] |
| 20:01 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.openstack.cloudvirt.lib.ensure_canary (exit_code=0) on eqiad1, with recreate False, for hosts list: ['cloudvirt1056'] |
[cloudvirt-canary] |
| 20:00 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.cloudvirt.lib.ensure_canary on eqiad1, with recreate False, for hosts list: ['cloudvirt1056'] |
[cloudvirt-canary] |
| 19:56 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_project_to_ovs |
[metricsinfra] |
| 19:55 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.openstack.migrate_project_to_ovs (exit_code=0) |
[wcdo] |
| 19:41 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_project_to_ovs |
[wcdo] |
| 19:40 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.openstack.migrate_project_to_ovs (exit_code=0) |
[wikisp] |
| 19:38 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_project_to_ovs |
[wikisp] |
| 19:35 |
<andrew@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudvirt1056.eqiad.wmnet with reason: host reimage |
[production] |
| 19:35 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.openstack.migrate_project_to_ovs (exit_code=0) |
[cloudinfra] |
| 19:32 |
<andrew@cumin1002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudvirt1056.eqiad.wmnet with reason: host reimage |
[production] |
| 19:27 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_project_to_ovs |
[cloudinfra] |
| 19:27 |
<bking@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow: apply |
[production] |
| 19:27 |
<bking@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow: apply |
[production] |
| 19:26 |
<bking@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow: apply |
[production] |
| 19:26 |
<bking@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow: apply |
[production] |
| 19:24 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.openstack.migrate_server_to_ovs (exit_code=99) for server fff879b9-8300-4a11-9cf6-3d424be9ffa3 |
[cloudinfra] |
| 19:24 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_server_to_ovs for server fff879b9-8300-4a11-9cf6-3d424be9ffa3 |
[cloudinfra] |
| 19:22 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.openstack.migrate_server_to_ovs (exit_code=99) for server fff879b9-8300-4a11-9cf6-3d424be9ffa3 |
[cloudinfra] |
| 19:20 |
<wmbot~lucaswerkmeister@tools-bastion-13> |
./bin/stashbot.sh restart # quit IRC a minute ago, following earlier reports of unspecified issues |
[tools.stashbot] |
| 19:12 |
<wmbot~lucaswerkmeister@tools-bastion-13> |
toolforge jobs restart bridgebot # *without* new image; bot had stopped bridging cloud IRC and telegram for no obvious reason and hopefully this fixes it |
[tools.bridgebot] |
| 19:11 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.openstack.cloudvirt.drain (exit_code=0) on host 'cloudvirt1056.eqiad.wmnet' |
[admin] |
| 19:10 |
<wmbot~lucaswerkmeister@tools-bastion-13> |
toolforge build delete bridgebot-buildpacks-pipelinerun-jtzrd # clear out failed build |
[tools.bridgebot] |
| 19:09 |
<wmbot~lucaswerkmeister@tools-bastion-13> |
toolforge build delete bridgebot-buildpacks-pipelinerun-bxnml # clear out more space, hopefully this [built from ref work/bd808/try-some-hacks in toolforge-repos/bridgebot-matterbridge] was no longer needed |
[tools.bridgebot] |
| 19:08 |
<mutante> |
LDAP - added daphnesmit to group 'wmf' - Phabricator: added dsmit-wmf to WMF-NDA group T368140 |
[production] |
| 19:07 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.openstack.migrate_server_to_ovs (exit_code=0) for server enc-2 |
[cloudinfra] |
| 19:06 |
<wmbot~lucaswerkmeister@tools-bastion-13> |
toolforge build delete bridgebot-buildpacks-pipelinerun-knkch bridgebot-buildpacks-pipelinerun-lvm2d bridgebot-buildpacks-pipelinerun-k4ddk bridgebot-buildpacks-pipelinerun-ghk8n # clear some storage space to let a new build fit in the quota |
[tools.bridgebot] |
| 19:06 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_server_to_ovs for server enc-2 |
[cloudinfra] |
| 19:05 |
<andrew@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.openstack.migrate_server_to_ovs (exit_code=99) for server enc-1 |
[cloudinfra] |
| 19:05 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_server_to_ovs for server enc-1 |
[cloudinfra] |
| 19:04 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.openstack.migrate_server_to_ovs (exit_code=0) for server cloudinfra-db03 |
[cloudinfra] |
| 19:03 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_server_to_ovs for server cloudinfra-db03 |
[cloudinfra] |
| 19:03 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.openstack.migrate_server_to_ovs (exit_code=0) for server cloudinfra-cloudvps-puppetserver-1 |
[cloudinfra] |
| 19:02 |
<sukhe> |
ms-fe1009: restart swift-proxy: T360913 |
[production] |
| 19:02 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_server_to_ovs for server cloudinfra-cloudvps-puppetserver-1 |
[cloudinfra] |
| 19:02 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.openstack.migrate_server_to_ovs (exit_code=0) for server cloudinfra-internal-puppetserver-1 |
[cloudinfra] |
| 19:01 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.openstack.migrate_server_to_ovs for server cloudinfra-internal-puppetserver-1 |
[cloudinfra] |