|
2025-02-01
ยง
|
| 16:50 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2146', diff saved to https://phabricator.wikimedia.org/P73004 and previous config saved to /var/cache/conftool/dbconfig/20250201-165055-marostegui.json |
[production] |
| 16:41 |
<cmooney@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3 days, 0:00:00 on cr2-magru with reason: IBGP instability from cr1 to cr2 in magru causing ping faulures from alert1002 |
[production] |
| 16:35 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2146', diff saved to https://phabricator.wikimedia.org/P73003 and previous config saved to /var/cache/conftool/dbconfig/20250201-163548-marostegui.json |
[production] |
| 16:20 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2146 (T384592)', diff saved to https://phabricator.wikimedia.org/P73002 and previous config saved to /var/cache/conftool/dbconfig/20250201-162041-marostegui.json |
[production] |
| 15:52 |
<wmbot~lucaswerkmeister@tools-bastion-13> |
kubectl rollout restart deployment bridgebot # stopped bridging for no clear reason (kubectl logs show recent telegram/IRC messages), see if this helps |
[tools.bridgebot] |
| 15:29 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for all nodes |
[toolsbeta] |
| 15:17 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2146 (T384592)', diff saved to https://phabricator.wikimedia.org/P73001 and previous config saved to /var/cache/conftool/dbconfig/20250201-151709-marostegui.json |
[production] |
| 15:17 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7:00:00 on db2146.codfw.wmnet with reason: Maintenance |
[production] |
| 15:16 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2145 (T384592)', diff saved to https://phabricator.wikimedia.org/P73000 and previous config saved to /var/cache/conftool/dbconfig/20250201-151646-marostegui.json |
[production] |
| 15:15 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for all nodes |
[toolsbeta] |
| 15:15 |
<andrew@cloudcumin1001> |
END (ERROR) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=97) for all nodes |
[toolsbeta] |
| 15:14 |
<andrewbogott> |
hard rebooting all VMs for T385264 |
[toolsbeta] |
| 15:14 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for all nodes |
[toolsbeta] |
| 15:06 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-108 |
[tools] |
| 15:05 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-108 |
[tools] |
| 15:05 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-107 |
[tools] |
| 15:04 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-107 |
[tools] |
| 15:04 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-106 |
[tools] |
| 15:03 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-106 |
[tools] |
| 15:03 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-105 |
[tools] |
| 15:02 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-105 |
[tools] |
| 15:02 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-103 |
[tools] |
| 15:01 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-103 |
[tools] |
| 15:01 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-102 |
[tools] |
| 15:01 |
<andrewbogott> |
rebooting all k8s (non-nfs) worker nodes for T385264 |
[tools] |
| 15:01 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2145', diff saved to https://phabricator.wikimedia.org/P72999 and previous config saved to /var/cache/conftool/dbconfig/20250201-150139-marostegui.json |
[production] |
| 15:00 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-102 |
[tools] |
| 14:57 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-9 |
[tools] |
| 14:56 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-9 |
[tools] |
| 14:56 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-74 |
[tools] |
| 14:55 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-74 |
[tools] |
| 14:55 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-71 |
[tools] |
| 14:53 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-71 |
[tools] |
| 14:53 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-66 |
[tools] |
| 14:48 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-66 |
[tools] |
| 14:48 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-54 |
[tools] |
| 14:47 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-54 |
[tools] |
| 14:47 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-50 |
[tools] |
| 14:46 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2145', diff saved to https://phabricator.wikimedia.org/P72998 and previous config saved to /var/cache/conftool/dbconfig/20250201-144632-marostegui.json |
[production] |
| 14:46 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-50 |
[tools] |
| 14:46 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-47 |
[tools] |
| 14:45 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-47 |
[tools] |
| 14:45 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-46 |
[tools] |
| 14:44 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-46 |
[tools] |
| 14:43 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-45 |
[tools] |
| 14:42 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-45 |
[tools] |
| 14:42 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-43 |
[tools] |
| 14:41 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-43 |
[tools] |
| 14:41 |
<andrew@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for tools-k8s-worker-nfs-40 |
[tools] |
| 14:40 |
<andrew@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for tools-k8s-worker-nfs-40 |
[tools] |