|
2026-07-21
ยง
|
| 13:03 |
<blake@deploy2003> |
helmfile [codfw] START helmfile.d/services/mw-pretrain: apply |
[production] |
| 13:02 |
<blake@deploy2003> |
helmfile [eqiad] DONE helmfile.d/services/mw-pretrain: apply |
[production] |
| 13:02 |
<blake@deploy2003> |
helmfile [eqiad] START helmfile.d/services/mw-pretrain: apply |
[production] |
| 12:58 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2217 (T431660)', diff saved to https://phabricator.wikimedia.org/P94992 and previous config saved to /var/cache/conftool/dbconfig/20260721-125801-cwilliams.json |
[production] |
| 12:53 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P94991 and previous config saved to /var/cache/conftool/dbconfig/20260721-125356-cwilliams.json |
[production] |
| 12:50 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Depooling db2217 (T431660)', diff saved to https://phabricator.wikimedia.org/P94990 and previous config saved to /var/cache/conftool/dbconfig/20260721-125049-cwilliams.json |
[production] |
| 12:50 |
<cwilliams@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2217.codfw.wmnet with reason: Maintenance |
[production] |
| 12:50 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2193 (T431660)', diff saved to https://phabricator.wikimedia.org/P94989 and previous config saved to /var/cache/conftool/dbconfig/20260721-125017-cwilliams.json |
[production] |
| 12:48 |
<elukey> |
bmc cold reboot for lvs1013 and lvs1015 - T426180 |
[production] |
| 12:43 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T431660)', diff saved to https://phabricator.wikimedia.org/P94988 and previous config saved to /var/cache/conftool/dbconfig/20260721-124348-cwilliams.json |
[production] |
| 12:40 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2193', diff saved to https://phabricator.wikimedia.org/P94987 and previous config saved to /var/cache/conftool/dbconfig/20260721-124009-cwilliams.json |
[production] |
| 12:35 |
<elukey@cumin1003> |
END (PASS) - Cookbook sre.hosts.bmc-user-mgmt (exit_code=0) for 1 hosts |
[production] |
| 12:34 |
<elukey@cumin1003> |
START - Cookbook sre.hosts.bmc-user-mgmt for 1 hosts |
[production] |
| 12:34 |
<elukey@cumin1003> |
END (PASS) - Cookbook sre.hosts.bmc-user-mgmt (exit_code=0) for 1 hosts |
[production] |
| 12:34 |
<elukey@cumin1003> |
START - Cookbook sre.hosts.bmc-user-mgmt for 1 hosts |
[production] |
| 12:34 |
<elukey@cumin1003> |
END (PASS) - Cookbook sre.hosts.bmc-user-mgmt (exit_code=0) for 3 hosts |
[production] |
| 12:33 |
<elukey@cumin1003> |
START - Cookbook sre.hosts.bmc-user-mgmt for 3 hosts |
[production] |
| 12:33 |
<elukey@cumin1003> |
END (PASS) - Cookbook sre.hosts.bmc-user-mgmt (exit_code=0) for 2 hosts |
[production] |
| 12:32 |
<elukey@cumin1003> |
START - Cookbook sre.hosts.bmc-user-mgmt for 2 hosts |
[production] |
| 12:31 |
<elukey@cumin1003> |
END (PASS) - Cookbook sre.hosts.bmc-user-mgmt (exit_code=0) for 1 hosts |
[production] |
| 12:31 |
<elukey@cumin1003> |
START - Cookbook sre.hosts.bmc-user-mgmt for 1 hosts |
[production] |
| 12:31 |
<elukey@cumin1003> |
END (PASS) - Cookbook sre.hosts.bmc-user-mgmt (exit_code=0) for 1 hosts |
[production] |
| 12:31 |
<elukey@cumin1003> |
START - Cookbook sre.hosts.bmc-user-mgmt for 1 hosts |
[production] |
| 12:30 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2193', diff saved to https://phabricator.wikimedia.org/P94986 and previous config saved to /var/cache/conftool/dbconfig/20260721-123001-cwilliams.json |
[production] |
| 12:19 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2193 (T431660)', diff saved to https://phabricator.wikimedia.org/P94985 and previous config saved to /var/cache/conftool/dbconfig/20260721-121953-cwilliams.json |
[production] |
| 12:17 |
<bking@cumin2003> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host wdqs2007.codfw.wmnet with OS bookworm |
[production] |
| 12:12 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Depooling db2193 (T431660)', diff saved to https://phabricator.wikimedia.org/P94983 and previous config saved to /var/cache/conftool/dbconfig/20260721-121257-cwilliams.json |
[production] |
| 12:12 |
<cwilliams@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2193.codfw.wmnet with reason: Maintenance |
[production] |
| 12:12 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2180 (T431660)', diff saved to https://phabricator.wikimedia.org/P94982 and previous config saved to /var/cache/conftool/dbconfig/20260721-121239-cwilliams.json |
[production] |
| 12:02 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2180', diff saved to https://phabricator.wikimedia.org/P94980 and previous config saved to /var/cache/conftool/dbconfig/20260721-120231-cwilliams.json |
[production] |
| 11:52 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2180', diff saved to https://phabricator.wikimedia.org/P94979 and previous config saved to /var/cache/conftool/dbconfig/20260721-115223-cwilliams.json |
[production] |
| 11:43 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Depooling db1180 (T431660)', diff saved to https://phabricator.wikimedia.org/P94978 and previous config saved to /var/cache/conftool/dbconfig/20260721-114333-cwilliams.json |
[production] |
| 11:43 |
<cwilliams@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1180.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:43 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T431660)', diff saved to https://phabricator.wikimedia.org/P94977 and previous config saved to /var/cache/conftool/dbconfig/20260721-114305-cwilliams.json |
[production] |
| 11:42 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2180 (T431660)', diff saved to https://phabricator.wikimedia.org/P94976 and previous config saved to /var/cache/conftool/dbconfig/20260721-114215-cwilliams.json |
[production] |
| 11:35 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Depooling db2180 (T431660)', diff saved to https://phabricator.wikimedia.org/P94975 and previous config saved to /var/cache/conftool/dbconfig/20260721-113530-cwilliams.json |
[production] |
| 11:35 |
<cwilliams@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2180.codfw.wmnet with reason: Maintenance |
[production] |
| 11:35 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2169 (T431660)', diff saved to https://phabricator.wikimedia.org/P94974 and previous config saved to /var/cache/conftool/dbconfig/20260721-113501-cwilliams.json |
[production] |
| 11:32 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P94973 and previous config saved to /var/cache/conftool/dbconfig/20260721-113258-cwilliams.json |
[production] |
| 11:24 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2169', diff saved to https://phabricator.wikimedia.org/P94972 and previous config saved to /var/cache/conftool/dbconfig/20260721-112453-cwilliams.json |
[production] |
| 11:22 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P94971 and previous config saved to /var/cache/conftool/dbconfig/20260721-112250-cwilliams.json |
[production] |
| 11:21 |
<XioNoX> |
put eqiad-drmrs Arelion link in service |
[production] |
| 11:14 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2169', diff saved to https://phabricator.wikimedia.org/P94970 and previous config saved to /var/cache/conftool/dbconfig/20260721-111446-cwilliams.json |
[production] |
| 11:12 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T431660)', diff saved to https://phabricator.wikimedia.org/P94969 and previous config saved to /var/cache/conftool/dbconfig/20260721-111242-cwilliams.json |
[production] |
| 11:10 |
<elukey@cumin1003> |
END (PASS) - Cookbook sre.hosts.bmc-user-mgmt (exit_code=0) for 1 hosts |
[production] |
| 11:10 |
<elukey@cumin1003> |
START - Cookbook sre.hosts.bmc-user-mgmt for 1 hosts |
[production] |
| 11:07 |
<elukey@cumin1003> |
END (PASS) - Cookbook sre.hosts.bmc-user-mgmt (exit_code=0) for 1093 hosts |
[production] |
| 11:05 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Depooling db1168 (T431660)', diff saved to https://phabricator.wikimedia.org/P94968 and previous config saved to /var/cache/conftool/dbconfig/20260721-110548-cwilliams.json |
[production] |
| 11:05 |
<cwilliams@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1168.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:05 |
<cwilliams@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1165 (T431660)', diff saved to https://phabricator.wikimedia.org/P94967 and previous config saved to /var/cache/conftool/dbconfig/20260721-110520-cwilliams.json |
[production] |