|
2024-05-29
ยง
|
| 12:25 |
<sfaci@deploy1002> |
helmfile [staging] START helmfile.d/services/media-analytics: apply |
[production] |
| 12:24 |
<jiji@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on mc1048.eqiad.wmnet with reason: host reimage |
[production] |
| 12:22 |
<jiji@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on mc2048.codfw.wmnet with reason: host reimage |
[production] |
| 12:22 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1170', diff saved to https://phabricator.wikimedia.org/P63527 and previous config saved to /var/cache/conftool/dbconfig/20240529-122239-marostegui.json |
[production] |
| 12:21 |
<sfaci@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/geo-analytics: apply |
[production] |
| 12:19 |
<sfaci@deploy1002> |
helmfile [eqiad] START helmfile.d/services/geo-analytics: apply |
[production] |
| 12:19 |
<slyngs> |
Failover idp.wikimedia.org for CAS upgrade to 6.6.15 |
[production] |
| 12:19 |
<jiji@cumin1002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on mc1048.eqiad.wmnet with reason: host reimage |
[production] |
| 12:18 |
<sfaci@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/geo-analytics: apply |
[production] |
| 12:17 |
<klausman@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ml-staging2002.codfw.wmnet with reason: host reimage |
[production] |
| 12:17 |
<sfaci@deploy1002> |
helmfile [codfw] START helmfile.d/services/geo-analytics: apply |
[production] |
| 12:16 |
<sfaci@deploy1002> |
helmfile [staging] DONE helmfile.d/services/geo-analytics: apply |
[production] |
| 12:15 |
<sfaci@deploy1002> |
helmfile [staging] START helmfile.d/services/geo-analytics: apply |
[production] |
| 12:14 |
<klausman@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on ml-staging2002.codfw.wmnet with reason: host reimage |
[production] |
| 12:12 |
<sfaci@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/page-analytics: apply |
[production] |
| 12:11 |
<sfaci@deploy1002> |
helmfile [eqiad] START helmfile.d/services/page-analytics: apply |
[production] |
| 12:10 |
<sfaci@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/page-analytics: apply |
[production] |
| 12:08 |
<sfaci@deploy1002> |
helmfile [codfw] START helmfile.d/services/page-analytics: apply |
[production] |
| 12:07 |
<sfaci@deploy1002> |
helmfile [staging] DONE helmfile.d/services/page-analytics: apply |
[production] |
| 12:07 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1170 (T366123)', diff saved to https://phabricator.wikimedia.org/P63526 and previous config saved to /var/cache/conftool/dbconfig/20240529-120730-marostegui.json |
[production] |
| 12:06 |
<sfaci@deploy1002> |
helmfile [staging] START helmfile.d/services/page-analytics: apply |
[production] |
| 12:05 |
<jiji@cumin1002> |
START - Cookbook sre.hosts.reimage for host mc1048.eqiad.wmnet with OS bookworm |
[production] |
| 12:04 |
<jiji@cumin2002> |
START - Cookbook sre.hosts.reimage for host mc2048.codfw.wmnet with OS bookworm |
[production] |
| 11:55 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.puppet.migrate-host (exit_code=0) for host db1235.eqiad.wmnet |
[production] |
| 11:53 |
<klausman@cumin2002> |
START - Cookbook sre.hosts.reimage for host ml-staging2002.codfw.wmnet with OS bookworm |
[production] |
| 11:50 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2122 (T364299)', diff saved to https://phabricator.wikimedia.org/P63525 and previous config saved to /var/cache/conftool/dbconfig/20240529-115051-marostegui.json |
[production] |
| 11:50 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2122.codfw.wmnet with reason: Maintenance |
[production] |
| 11:50 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2122.codfw.wmnet with reason: Maintenance |
[production] |
| 11:50 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2121 (T364299)', diff saved to https://phabricator.wikimedia.org/P63524 and previous config saved to /var/cache/conftool/dbconfig/20240529-115025-marostegui.json |
[production] |
| 11:46 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.idm.logout (exit_code=0) Logging Mabualruz out of all services on: 2198 hosts |
[production] |
| 11:46 |
<jmm@cumin2002> |
START - Cookbook sre.idm.logout Logging Mabualruz out of all services on: 2198 hosts |
[production] |
| 11:44 |
<sfaci@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/device-analytics: apply |
[production] |
| 11:42 |
<sfaci@deploy1002> |
helmfile [eqiad] START helmfile.d/services/device-analytics: apply |
[production] |
| 11:42 |
<marostegui> |
recreate triggers on s7 eqiad db maint db1155:3317 T366167 |
[production] |
| 11:41 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1170 (T366123)', diff saved to https://phabricator.wikimedia.org/P63523 and previous config saved to /var/cache/conftool/dbconfig/20240529-114153-marostegui.json |
[production] |
| 11:41 |
<hnowlan> |
homer "cr*eqiad*" commit 'adding bgp state for wikikube-ctrl1002' |
[production] |
| 11:41 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:41 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:41 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1158 (T366123)', diff saved to https://phabricator.wikimedia.org/P63522 and previous config saved to /var/cache/conftool/dbconfig/20240529-114129-marostegui.json |
[production] |
| 11:40 |
<sfaci@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/device-analytics: apply |
[production] |
| 11:38 |
<sfaci@deploy1002> |
helmfile [codfw] START helmfile.d/services/device-analytics: apply |
[production] |
| 11:35 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2121', diff saved to https://phabricator.wikimedia.org/P63521 and previous config saved to /var/cache/conftool/dbconfig/20240529-113517-marostegui.json |
[production] |
| 11:26 |
<sfaci@deploy1002> |
helmfile [staging] DONE helmfile.d/services/device-analytics: apply |
[production] |
| 11:26 |
<akosiaris@cumin1002> |
conftool action : set/pooled=yes; selector: service=kubemaster,dc=eqiad,cluster=kubernetes,name=wikikube-ctrl1001.eqiad.wmnet |
[production] |
| 11:26 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1158', diff saved to https://phabricator.wikimedia.org/P63520 and previous config saved to /var/cache/conftool/dbconfig/20240529-112621-marostegui.json |
[production] |
| 11:26 |
<akosiaris@cumin1002> |
conftool action : set/pooled=yes; selector: service=kubemaster,dc=eqiad,cluster=kubernetes,name=kubemaster1002.eqiad.wmnet |
[production] |
| 11:25 |
<sfaci@deploy1002> |
helmfile [staging] START helmfile.d/services/device-analytics: apply |
[production] |
| 11:23 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 11:23 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 11:23 |
<akosiaris> |
T366094 re-undeploy otel-collector, it being around increased traffic to the API >50% |
[production] |