|
2022-06-13
ยง
|
| 11:07 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host pki2002.codfw.wmnet |
[production] |
| 11:04 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetboard1002.eqiad.wmnet |
[production] |
| 11:03 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetboard2002.codfw.wmnet |
[production] |
| 11:02 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.idm.logout (exit_code=0) Logging Dsharpe out of all services on: 1219 hosts |
[production] |
| 11:00 |
<jmm@cumin2002> |
START - Cookbook sre.idm.logout Logging Dsharpe out of all services on: 1219 hosts |
[production] |
| 11:00 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host puppetboard2002.codfw.wmnet |
[production] |
| 10:59 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1143', diff saved to https://phabricator.wikimedia.org/P29643 and previous config saved to /var/cache/conftool/dbconfig/20220613-105954-marostegui.json |
[production] |
| 10:56 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.idm.logout (exit_code=0) Logging Dsharpe out of all services on: 609 hosts |
[production] |
| 10:56 |
<jmm@cumin2002> |
START - Cookbook sre.idm.logout Logging Dsharpe out of all services on: 609 hosts |
[production] |
| 10:52 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 10:52 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 10:52 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 10:51 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 10:51 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 10:50 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 10:50 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 10:50 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 10:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1143 (T310011)', diff saved to https://phabricator.wikimedia.org/P29642 and previous config saved to /var/cache/conftool/dbconfig/20220613-104449-marostegui.json |
[production] |
| 10:38 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 10:38 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 10:37 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 10:37 |
<klausman@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 10:15 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1143 (T310011)', diff saved to https://phabricator.wikimedia.org/P29641 and previous config saved to /var/cache/conftool/dbconfig/20220613-101537-marostegui.json |
[production] |
| 10:15 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1143.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:15 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1143.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:13 |
<moritzm> |
installing 5.10.120 kernel updates on bullseye hosts |
[production] |
| 09:53 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1150.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:53 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1150.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:12 |
<moritzm> |
drain ganeti3001 for firmware update/reimage T308238 |
[production] |
| 09:07 |
<moritzm> |
installing ntfs-3g security updates |
[production] |
| 07:54 |
<moritzm> |
failover ganeti master in esams to ganeti3003 T308238 |
[production] |
| 07:18 |
<joal> |
Manually rerun webrequest_text laod for hour 2022-06-12T08:00 |
[production] |
| 06:41 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1172 (re)pooling @ 100%: After schema change', diff saved to https://phabricator.wikimedia.org/P29640 and previous config saved to /var/cache/conftool/dbconfig/20220613-064109-root.json |
[production] |
| 06:26 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1172 (re)pooling @ 75%: After schema change', diff saved to https://phabricator.wikimedia.org/P29639 and previous config saved to /var/cache/conftool/dbconfig/20220613-062605-root.json |
[production] |
| 06:11 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1172 (re)pooling @ 50%: After schema change', diff saved to https://phabricator.wikimedia.org/P29638 and previous config saved to /var/cache/conftool/dbconfig/20220613-061101-root.json |
[production] |
| 05:55 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1172 (re)pooling @ 25%: After schema change', diff saved to https://phabricator.wikimedia.org/P29637 and previous config saved to /var/cache/conftool/dbconfig/20220613-055557-root.json |
[production] |
| 05:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1172', diff saved to https://phabricator.wikimedia.org/P29636 and previous config saved to /var/cache/conftool/dbconfig/20220613-054623-marostegui.json |
[production] |
| 05:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1172', diff saved to https://phabricator.wikimedia.org/P29635 and previous config saved to /var/cache/conftool/dbconfig/20220613-053118-marostegui.json |
[production] |
| 05:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1172 (T310011)', diff saved to https://phabricator.wikimedia.org/P29634 and previous config saved to /var/cache/conftool/dbconfig/20220613-051613-marostegui.json |
[production] |
| 05:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1172 (T310011)', diff saved to https://phabricator.wikimedia.org/P29633 and previous config saved to /var/cache/conftool/dbconfig/20220613-051407-marostegui.json |
[production] |
| 05:14 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1172.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:14 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1172.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:14 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1171.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:13 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1171.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:59 |
<kart_> |
Updated cxserver to 2022-06-08-124326-production + nodejs > node command update (T306995, T309169) |
[production] |
| 04:57 |
<kartik@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/cxserver: apply |
[production] |
| 04:56 |
<kartik@deploy1002> |
helmfile [eqiad] START helmfile.d/services/cxserver: apply |
[production] |
| 04:54 |
<kartik@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/cxserver: apply |
[production] |
| 04:54 |
<kartik@deploy1002> |
helmfile [codfw] START helmfile.d/services/cxserver: apply |
[production] |
| 04:50 |
<kartik@deploy1002> |
helmfile [staging] DONE helmfile.d/services/cxserver: apply |
[production] |