|
2026-10-08
ยง
|
| 09:11 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-worker2161.codfw.wmnet |
[production] |
| 09:10 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker2160.codfw.wmnet |
[production] |
| 09:10 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker2160.codfw.wmnet |
[production] |
| 09:10 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2160.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 09:00 |
<oblivian@cumin1004> |
START - Cookbook sre.hosts.provision for host wikikube-worker2160.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:59 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host wikikube-worker2160.codfw.wmnet |
[production] |
| 08:59 |
<moritzm> |
installing Wireshark security updates |
[production] |
| 08:59 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-worker2160.codfw.wmnet |
[production] |
| 08:59 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker2159.codfw.wmnet |
[production] |
| 08:57 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker2159.codfw.wmnet |
[production] |
| 08:57 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2159.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:53 |
<marostegui@cumin1004> |
START - Cookbook sre.mysql.pool pool db2191: Repooling after switchover |
[production] |
| 08:53 |
<marostegui@cumin1004> |
END (ERROR) - Cookbook sre.mysql.pool (exit_code=97) pool db2191: Repooling after switchover |
[production] |
| 08:53 |
<marostegui@cumin1004> |
START - Cookbook sre.mysql.pool pool db2191: Repooling after switchover |
[production] |
| 08:52 |
<marostegui@cumin1004> |
dbctl commit (dc=all): 'Depool db2191 T436922', diff saved to https://phabricator.wikimedia.org/P96627 and previous config saved to /var/cache/conftool/dbconfig/20261008-085238-marostegui.json |
[production] |
| 08:50 |
<marostegui@cumin1004> |
dbctl commit (dc=all): 'Promote db2249 to x1 primary T436922', diff saved to https://phabricator.wikimedia.org/P96626 and previous config saved to /var/cache/conftool/dbconfig/20261008-085050-marostegui.json |
[production] |
| 08:50 |
<marostegui> |
Starting x1 codfw failover from db2191 to db2249 - T436922 |
[production] |
| 08:46 |
<marostegui@cumin1004> |
dbctl commit (dc=all): 'Set db2249 with weight 0 T436922', diff saved to https://phabricator.wikimedia.org/P96625 and previous config saved to /var/cache/conftool/dbconfig/20261008-084653-marostegui.json |
[production] |
| 08:46 |
<marostegui@cumin1004> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 17 hosts with reason: Primary switchover x1 T436922 |
[production] |
| 08:46 |
<root@deploy1003> |
helmfile [staging] DONE helmfile.d/services/shellbox-timeline: sync |
[production] |
| 08:46 |
<root@deploy1003> |
helmfile [staging] START helmfile.d/services/shellbox-timeline: sync |
[production] |
| 08:46 |
<oblivian@cumin1004> |
START - Cookbook sre.hosts.provision for host wikikube-worker1057.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:46 |
<root@deploy1003> |
helmfile [staging] DONE helmfile.d/services/shellbox: sync |
[production] |
| 08:46 |
<root@deploy1003> |
helmfile [staging] START helmfile.d/services/shellbox: sync |
[production] |
| 08:46 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host wikikube-worker1057.eqiad.wmnet |
[production] |
| 08:46 |
<oblivian@cumin1004> |
START - Cookbook sre.hosts.provision for host wikikube-worker2159.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:46 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host wikikube-worker2159.codfw.wmnet |
[production] |
| 08:45 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-worker2159.codfw.wmnet |
[production] |
| 08:45 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker2158.codfw.wmnet |
[production] |
| 08:45 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-worker1057.eqiad.wmnet |
[production] |
| 08:44 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker1056.eqiad.wmnet |
[production] |
| 08:43 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker2158.codfw.wmnet |
[production] |
| 08:42 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker2158.mgmt.codfw.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:42 |
<moritzm> |
installing python3.13 security updates |
[production] |
| 08:42 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker1056.eqiad.wmnet |
[production] |
| 08:42 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1056.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:33 |
<oblivian@cumin1004> |
START - Cookbook sre.hosts.provision for host wikikube-worker1056.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:33 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host wikikube-worker1056.eqiad.wmnet |
[production] |
| 08:32 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-worker1056.eqiad.wmnet |
[production] |
| 08:32 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker1055.eqiad.wmnet |
[production] |
| 08:32 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker1055.eqiad.wmnet |
[production] |
| 08:32 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1055.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:24 |
<oblivian@cumin1004> |
START - Cookbook sre.hosts.provision for host wikikube-worker1055.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:23 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) depool for host wikikube-worker1055.eqiad.wmnet |
[production] |
| 08:23 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node depool for host wikikube-worker1055.eqiad.wmnet |
[production] |
| 08:23 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.k8s.pool-depool-node (exit_code=0) pool for host wikikube-worker1054.eqiad.wmnet |
[production] |
| 08:22 |
<oblivian@cumin1004> |
START - Cookbook sre.k8s.pool-depool-node pool for host wikikube-worker1054.eqiad.wmnet |
[production] |
| 08:22 |
<oblivian@cumin1004> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host wikikube-worker1054.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |
| 08:20 |
<aklapper@deploy1003> |
rebuilt and synchronized wikiversions files: group2 to 1.47.0-wmf.23 refs T438219 |
[production] |
| 08:14 |
<oblivian@cumin1004> |
START - Cookbook sre.hosts.provision for host wikikube-worker1054.mgmt.eqiad.wmnet with chassis set policy GRACEFUL_RESTART and with Dell SCP reboot policy GRACEFUL |
[production] |