|
2023-10-18
§
|
| 11:38 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-71 from 1.22.17 to 1.23.17 |
[tools] |
| 11:37 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-51 from 1.22.17 to 1.23.17 |
[tools] |
| 11:37 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-33 from 1.22.17 to 1.23.17 |
[tools] |
| 11:35 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-32 from 1.22.17 to 1.23.17 |
[tools] |
| 11:34 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-32 from 1.22.17 to 1.23.17 |
[tools] |
| 11:34 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-31 from 1.22.17 to 1.23.17 |
[tools] |
| 11:32 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-31 from 1.22.17 to 1.23.17 |
[tools] |
| 11:31 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-worker-30 from 1.22.17 to 1.23.17 |
[tools] |
| 11:30 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-worker-30 from 1.22.17 to 1.23.17 |
[tools] |
| 11:29 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-control-6 from 1.22.17 to 1.23.17 |
[tools] |
| 11:25 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-control-6 from 1.22.17 to 1.23.17 |
[tools] |
| 11:23 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-control-5 from 1.22.17 to 1.23.17 |
[tools] |
| 11:16 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-control-5 from 1.22.17 to 1.23.17 |
[tools] |
| 11:15 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.worker.upgrade (exit_code=0) for node tools-k8s-control-4 from 1.22.17 to 1.23.17 |
[tools] |
| 11:07 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.worker.upgrade for node tools-k8s-control-4 from 1.22.17 to 1.23.17 |
[tools] |
| 11:04 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.prepare_upgrade (exit_code=0) for cluster tools upgrade from 1.22.17 to 1.23.17 (T298005) |
[tools] |
| 11:03 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.prepare_upgrade for cluster tools upgrade from 1.22.17 to 1.23.17 (T298005) |
[tools] |
|
2023-10-13
§
|
| 13:29 |
<wm-bot2> |
dcaro@urcuchillay END (PASS) - Cookbook wmcs.toolforge.k8s.component.deploy (exit_code=0) for component builds-api |
[tools] |
| 13:28 |
<wm-bot2> |
dcaro@urcuchillay START - Cookbook wmcs.toolforge.k8s.component.deploy for component builds-api |
[tools] |
| 09:48 |
<sstefanova@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.component.deploy (exit_code=0) for component builds-builder |
[tools] |
| 09:48 |
<sstefanova@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.component.deploy for component builds-builder |
[tools] |
| 09:07 |
<wm-bot2> |
dcaro@urcuchillay END (PASS) - Cookbook wmcs.toolforge.grid.cleanup_queue_errors (exit_code=0) |
[tools] |
| 09:07 |
<wm-bot2> |
dcaro@urcuchillay START - Cookbook wmcs.toolforge.grid.cleanup_queue_errors |
[tools] |
| 08:57 |
<dcaro> |
rebooting tools-sgeexec-10-8 as the host is stuck/unreachable |
[tools] |
| 07:43 |
<dcaro> |
rebooting tools-sgeweblight-10-26 as it fails to allocate memory |
[tools] |
|
2023-10-12
§
|
| 15:07 |
<taavi> |
reboot tools-k8s-worker-70 |
[tools] |
| 14:01 |
<taavi> |
deploy jobs-cli v15 T348250 |
[tools] |
| 13:10 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.apt.copy_to_main_repo (exit_code=0) for package 'toolforge-jobs-framework-cli' version '15' |
[tools] |
| 13:10 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.apt.copy_to_main_repo for package 'toolforge-jobs-framework-cli' version '15' |
[tools] |
| 12:21 |
<dcaro> |
rebooting sgeexec-10-17 |
[tools] |
| 12:02 |
<taavi> |
also reboot tools-sgeweblight-10-30 |
[tools] |
| 12:00 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.grid.cleanup_queue_errors (exit_code=0) |
[tools] |
| 12:00 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.grid.cleanup_queue_errors |
[tools] |
| 11:52 |
<taavi> |
reboot tools-sgeweblight-10-22, 28 |
[tools] |
|
2023-10-11
§
|
| 19:47 |
<taavi@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.k8s.reboot (exit_code=0) for all workers |
[tools] |
| 17:10 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.reboot for all workers |
[tools] |
| 14:41 |
<wm-bot2> |
dcaro@urcuchillay END (FAIL) - Cookbook wmcs.toolforge.grid.reboot_workers (exit_code=99) for weblight nodes (T348634) |
[tools] |
| 14:24 |
<wm-bot2> |
dcaro@urcuchillay START - Cookbook wmcs.toolforge.grid.reboot_workers for weblight nodes (T348634) |
[tools] |
| 14:21 |
<wm-bot2> |
dcaro@urcuchillay END (PASS) - Cookbook wmcs.openstack.cloudvirt.vm_console (exit_code=0) |
[tools] |
| 14:20 |
<wm-bot2> |
dcaro@urcuchillay START - Cookbook wmcs.openstack.cloudvirt.vm_console |
[tools] |
| 14:19 |
<wm-bot2> |
dcaro@urcuchillay END (PASS) - Cookbook wmcs.openstack.cloudvirt.vm_console (exit_code=0) |
[tools] |
| 14:19 |
<wm-bot2> |
dcaro@urcuchillay START - Cookbook wmcs.openstack.cloudvirt.vm_console |
[tools] |
| 14:16 |
<dcaro> |
rebooting tools-sgeweblight-10-16 due to stuck NFS (T348634) |
[tools] |
| 12:11 |
<taavi> |
reboot k8s workers 48, 60, 65, 68, 70, 76 T348634 |
[tools] |
| 12:04 |
<taavi> |
reboot k8s workers 72, 75, 82 T348634 |
[tools] |
| 12:01 |
<taavi> |
reboot tools-sgecron-2 T348634 |
[tools] |
| 11:49 |
<taavi> |
reboot tools-sgeexec-10-19 |
[tools] |