|
2025-11-06
ยง
|
| 18:09 |
<swfrench@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-api-ext: apply |
[production] |
| 18:09 |
<bd808@deploy2002> |
helmfile [eqiad] START helmfile.d/services/developer-portal: apply |
[production] |
| 18:09 |
<swfrench@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-ext: apply |
[production] |
| 18:09 |
<swfrench@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-api-ext: apply |
[production] |
| 18:07 |
<swfrench@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: apply |
[production] |
| 18:06 |
<swfrench@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-web: apply |
[production] |
| 18:06 |
<swfrench@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-ext: apply |
[production] |
| 18:06 |
<robh> |
Rack C2 C/D switch migrations in progress |
[production] |
| 18:06 |
<swfrench@deploy2002> |
helmfile [eqiad] START helmfile.d/services/mw-api-ext: apply |
[production] |
| 18:05 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on ms-be1092.eqiad.wmnet with reason: C/D Migration |
[production] |
| 18:04 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on ms-be1086.eqiad.wmnet with reason: C/D Migration |
[production] |
| 18:02 |
<bd808@deploy2002> |
helmfile [staging] DONE helmfile.d/services/developer-portal: apply |
[production] |
| 18:02 |
<bd808@deploy2002> |
helmfile [staging] START helmfile.d/services/developer-portal: apply |
[production] |
| 18:01 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:10:00 on ms-be1066.eqiad.wmnet with reason: C/D Migration |
[production] |
| 18:00 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Depooling db1235 (T407997)', diff saved to https://phabricator.wikimedia.org/P85041 and previous config saved to /var/cache/conftool/dbconfig/20251106-180052-marostegui.json |
[production] |
| 18:00 |
<marostegui@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db1235.eqiad.wmnet with reason: Maintenance |
[production] |
| 18:00 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1234 (T407997)', diff saved to https://phabricator.wikimedia.org/P85040 and previous config saved to /var/cache/conftool/dbconfig/20251106-180028-marostegui.json |
[production] |
| 17:53 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on apus-fe1003.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:51 |
<fceratto@cumin1003> |
END (PASS) - Cookbook sre.mysql.major-upgrade (exit_code=0) |
[production] |
| 17:51 |
<fceratto@cumin1003> |
END (PASS) - Cookbook sre.mysql.pool (exit_code=0) db2154 gradually with 4 steps - Migration of db2154.codfw.wmnet completed |
[production] |
| 17:50 |
<dzahn@cumin2002> |
END (FAIL) - Cookbook sre.hosts.decommission (exit_code=99) for hosts people2003.codfw.wmnet |
[production] |
| 17:47 |
<jhancock@cumin1003> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2091.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |
| 17:46 |
<jhancock@cumin1003> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2092.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |
| 17:46 |
<jhancock@cumin1003> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2093.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |
| 17:46 |
<jhancock@cumin1003> |
END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host ms-be2090.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |
| 17:45 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1234', diff saved to https://phabricator.wikimedia.org/P85038 and previous config saved to /var/cache/conftool/dbconfig/20251106-174521-marostegui.json |
[production] |
| 17:42 |
<dzahn@cumin2002> |
START - Cookbook sre.hosts.decommission for hosts people2003.codfw.wmnet |
[production] |
| 17:39 |
<dzahn@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 14 days, 0:00:00 on people1004.eqiad.wmnet with reason: decom |
[production] |
| 17:39 |
<dzahn@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 14 days, 0:00:00 on people2003.codfw.wmnet with reason: decom |
[production] |
| 17:38 |
<mutante> |
shutting down people1004 and people2003 - had already shut them down on Oct 29 but someone or something booted them again T408713 |
[production] |
| 17:37 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on cp1110.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:33 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on mc1050.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:31 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on mc1049.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:30 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1234', diff saved to https://phabricator.wikimedia.org/P85036 and previous config saved to /var/cache/conftool/dbconfig/20251106-173013-marostegui.json |
[production] |
| 17:28 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on wdqs1014.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:25 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on relforge1008.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:23 |
<jhancock@cumin1003> |
END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host ms-be2094.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |
| 17:21 |
<robh> |
multiple moves from C/D per T405942 |
[production] |
| 17:19 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on cirrussearch1119.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:19 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on cirrussearch1118.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:17 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on cirrussearch1081.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:16 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on cirrussearch1080.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:15 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1234 (T407997)', diff saved to https://phabricator.wikimedia.org/P85034 and previous config saved to /var/cache/conftool/dbconfig/20251106-171505-marostegui.json |
[production] |
| 17:14 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on ml-cache1002.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:13 |
<jhancock@cumin1003> |
START - Cookbook sre.hosts.provision for host ms-be2094.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |
| 17:12 |
<jhancock@cumin1003> |
START - Cookbook sre.hosts.provision for host ms-be2093.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |
| 17:12 |
<robh@cumin2002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on an-worker1223.eqiad.wmnet with reason: C/D Migration |
[production] |
| 17:12 |
<jhancock@cumin1003> |
START - Cookbook sre.hosts.provision for host ms-be2092.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |
| 17:11 |
<jhancock@cumin1003> |
START - Cookbook sre.hosts.provision for host ms-be2091.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |
| 17:11 |
<jhancock@cumin1003> |
START - Cookbook sre.hosts.provision for host ms-be2090.mgmt.codfw.wmnet with chassis set policy FORCE_RESTART and with Dell SCP reboot policy FORCED |
[production] |