|
2024-05-29
ยง
|
| 11:50 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2122.codfw.wmnet with reason: Maintenance |
[production] |
| 11:50 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2121 (T364299)', diff saved to https://phabricator.wikimedia.org/P63524 and previous config saved to /var/cache/conftool/dbconfig/20240529-115025-marostegui.json |
[production] |
| 11:46 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.idm.logout (exit_code=0) Logging Mabualruz out of all services on: 2198 hosts |
[production] |
| 11:46 |
<jmm@cumin2002> |
START - Cookbook sre.idm.logout Logging Mabualruz out of all services on: 2198 hosts |
[production] |
| 11:44 |
<sfaci@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/device-analytics: apply |
[production] |
| 11:42 |
<sfaci@deploy1002> |
helmfile [eqiad] START helmfile.d/services/device-analytics: apply |
[production] |
| 11:42 |
<marostegui> |
recreate triggers on s7 eqiad db maint db1155:3317 T366167 |
[production] |
| 11:41 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1170 (T366123)', diff saved to https://phabricator.wikimedia.org/P63523 and previous config saved to /var/cache/conftool/dbconfig/20240529-114153-marostegui.json |
[production] |
| 11:41 |
<hnowlan> |
homer "cr*eqiad*" commit 'adding bgp state for wikikube-ctrl1002' |
[production] |
| 11:41 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:41 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:41 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1158 (T366123)', diff saved to https://phabricator.wikimedia.org/P63522 and previous config saved to /var/cache/conftool/dbconfig/20240529-114129-marostegui.json |
[production] |
| 11:40 |
<sfaci@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/device-analytics: apply |
[production] |
| 11:38 |
<sfaci@deploy1002> |
helmfile [codfw] START helmfile.d/services/device-analytics: apply |
[production] |
| 11:35 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2121', diff saved to https://phabricator.wikimedia.org/P63521 and previous config saved to /var/cache/conftool/dbconfig/20240529-113517-marostegui.json |
[production] |
| 11:26 |
<sfaci@deploy1002> |
helmfile [staging] DONE helmfile.d/services/device-analytics: apply |
[production] |
| 11:26 |
<akosiaris@cumin1002> |
conftool action : set/pooled=yes; selector: service=kubemaster,dc=eqiad,cluster=kubernetes,name=wikikube-ctrl1001.eqiad.wmnet |
[production] |
| 11:26 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1158', diff saved to https://phabricator.wikimedia.org/P63520 and previous config saved to /var/cache/conftool/dbconfig/20240529-112621-marostegui.json |
[production] |
| 11:26 |
<akosiaris@cumin1002> |
conftool action : set/pooled=yes; selector: service=kubemaster,dc=eqiad,cluster=kubernetes,name=kubemaster1002.eqiad.wmnet |
[production] |
| 11:25 |
<sfaci@deploy1002> |
helmfile [staging] START helmfile.d/services/device-analytics: apply |
[production] |
| 11:23 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 11:23 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 11:23 |
<akosiaris> |
T366094 re-undeploy otel-collector, it being around increased traffic to the API >50% |
[production] |
| 11:20 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2121', diff saved to https://phabricator.wikimedia.org/P63519 and previous config saved to /var/cache/conftool/dbconfig/20240529-112009-marostegui.json |
[production] |
| 11:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-ext: sync |
[production] |
| 11:16 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: sync |
[production] |
| 11:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-int: sync |
[production] |
| 11:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-parsoid: sync |
[production] |
| 11:12 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-debug: sync |
[production] |
| 11:11 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1158', diff saved to https://phabricator.wikimedia.org/P63518 and previous config saved to /var/cache/conftool/dbconfig/20240529-111112-marostegui.json |
[production] |
| 11:10 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-wikifunctions: sync |
[production] |
| 11:10 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-web: sync |
[production] |
| 11:10 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-api-ext: sync |
[production] |
| 11:10 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-debug: sync |
[production] |
| 11:10 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-api-int: sync |
[production] |
| 11:10 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-parsoid: sync |
[production] |
| 11:10 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-wikifunctions: sync |
[production] |
| 11:06 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 11:06 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 11:05 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2121 (T364299)', diff saved to https://phabricator.wikimedia.org/P63517 and previous config saved to /var/cache/conftool/dbconfig/20240529-110501-marostegui.json |
[production] |
| 11:04 |
<akosiaris> |
redeploy opentelemetry collector T366094 |
[production] |
| 11:03 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 11:03 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 11:03 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 11:03 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 11:03 |
<jiji@cumin1002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host mc1049.eqiad.wmnet with OS bookworm |
[production] |
| 11:03 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-ext: sync |
[production] |
| 10:57 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-int: sync |
[production] |
| 10:57 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-parsoid: sync |
[production] |
| 10:56 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: sync |
[production] |