|
2025-11-24
ยง
|
| 11:23 |
<marostegui@cumin1003> |
START - Cookbook sre.mysql.pool db1187 gradually with 4 steps - repool after schema change test |
[production] |
| 11:23 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Depooling db1212 (T410531)', diff saved to https://phabricator.wikimedia.org/P85506 and previous config saved to /var/cache/conftool/dbconfig/20251124-112306-marostegui.json |
[production] |
| 11:22 |
<marostegui@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on 6 hosts with reason: Maintenance |
[production] |
| 11:22 |
<marostegui@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db1212.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:22 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1198 (T410531)', diff saved to https://phabricator.wikimedia.org/P85505 and previous config saved to /var/cache/conftool/dbconfig/20251124-112231-marostegui.json |
[production] |
| 11:21 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Depooling db1187 (T299441)', diff saved to https://phabricator.wikimedia.org/P85504 and previous config saved to /var/cache/conftool/dbconfig/20251124-112111-marostegui.json |
[production] |
| 11:21 |
<marostegui@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db1187.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:07 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1198', diff saved to https://phabricator.wikimedia.org/P85503 and previous config saved to /var/cache/conftool/dbconfig/20251124-110723-marostegui.json |
[production] |
| 10:57 |
<cgoubert@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/rest-gateway: apply |
[production] |
| 10:56 |
<cgoubert@deploy2002> |
helmfile [codfw] START helmfile.d/services/rest-gateway: apply |
[production] |
| 10:56 |
<cgoubert@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/rest-gateway: apply |
[production] |
| 10:56 |
<cgoubert@deploy2002> |
helmfile [eqiad] START helmfile.d/services/rest-gateway: apply |
[production] |
| 10:56 |
<btullis@deploy2002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/growthbook: apply |
[production] |
| 10:56 |
<btullis@deploy2002> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/growthbook: apply |
[production] |
| 10:56 |
<cgoubert@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/rest-gateway: apply |
[production] |
| 10:55 |
<cgoubert@deploy2002> |
helmfile [eqiad] START helmfile.d/services/rest-gateway: apply |
[production] |
| 10:52 |
<cgoubert@deploy2002> |
helmfile [codfw] DONE helmfile.d/services/api-gateway: apply |
[production] |
| 10:52 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1198', diff saved to https://phabricator.wikimedia.org/P85502 and previous config saved to /var/cache/conftool/dbconfig/20251124-105216-marostegui.json |
[production] |
| 10:51 |
<cgoubert@deploy2002> |
helmfile [codfw] START helmfile.d/services/api-gateway: apply |
[production] |
| 10:51 |
<cgoubert@deploy2002> |
helmfile [eqiad] DONE helmfile.d/services/api-gateway: apply |
[production] |
| 10:51 |
<cgoubert@deploy2002> |
helmfile [eqiad] START helmfile.d/services/api-gateway: apply |
[production] |
| 10:48 |
<btullis@deploy2002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/growthbook: apply |
[production] |
| 10:48 |
<gehel@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-cluster (exit_code=0) |
[production] |
| 10:47 |
<btullis@deploy2002> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/growthbook: apply |
[production] |
| 10:46 |
<gehel@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-cluster (exit_code=0) |
[production] |
| 10:46 |
<claime> |
Deploying envoy 1.32 to api-gateway |
[production] |
| 10:39 |
<gehel@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-cluster (exit_code=0) |
[production] |
| 10:37 |
<btullis@deploy2002> |
helmfile [dse-k8s-codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 10:37 |
<btullis@deploy2002> |
helmfile [dse-k8s-codfw] START helmfile.d/admin 'apply'. |
[production] |
| 10:37 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1198 (T410531)', diff saved to https://phabricator.wikimedia.org/P85501 and previous config saved to /var/cache/conftool/dbconfig/20251124-103708-marostegui.json |
[production] |
| 10:36 |
<btullis@deploy2002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 10:36 |
<btullis@deploy2002> |
helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 10:34 |
<gehel@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-cluster (exit_code=0) |
[production] |
| 10:32 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Depooling db1198 (T410531)', diff saved to https://phabricator.wikimedia.org/P85500 and previous config saved to /var/cache/conftool/dbconfig/20251124-103218-marostegui.json |
[production] |
| 10:32 |
<marostegui@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on db1198.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:31 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1189 (T410531)', diff saved to https://phabricator.wikimedia.org/P85499 and previous config saved to /var/cache/conftool/dbconfig/20251124-103155-marostegui.json |
[production] |
| 10:27 |
<gehel@cumin2002> |
START - Cookbook sre.hosts.reboot-cluster |
[production] |
| 10:26 |
<gehel@cumin1003> |
START - Cookbook sre.hosts.reboot-cluster |
[production] |
| 10:25 |
<gehel@cumin2002> |
START - Cookbook sre.hosts.reboot-cluster |
[production] |
| 10:25 |
<gehel@cumin1003> |
START - Cookbook sre.hosts.reboot-cluster |
[production] |
| 10:23 |
<volans@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.component.deploy (exit_code=0) for component infra-tracing |
[tools] |
| 10:22 |
<gehel@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-cluster (exit_code=0) |
[production] |
| 10:16 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1189', diff saved to https://phabricator.wikimedia.org/P85498 and previous config saved to /var/cache/conftool/dbconfig/20251124-101647-marostegui.json |
[production] |
| 10:14 |
<gehel@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-cluster (exit_code=0) |
[production] |
| 10:14 |
<gehel@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-cluster (exit_code=0) |
[production] |
| 10:04 |
<volans@cloudcumin1001> |
START - Cookbook wmcs.toolforge.component.deploy for component infra-tracing |
[tools] |
| 10:03 |
<volans@cloudcumin1001> |
END (PASS) - Cookbook wmcs.toolforge.component.deploy (exit_code=0) for component infra-tracing |
[toolsbeta] |
| 10:01 |
<marostegui@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1189', diff saved to https://phabricator.wikimedia.org/P85497 and previous config saved to /var/cache/conftool/dbconfig/20251124-100139-marostegui.json |
[production] |
| 10:00 |
<gehel@cumin2002> |
START - Cookbook sre.hosts.reboot-cluster |
[production] |
| 09:59 |
<gehel@cumin1003> |
START - Cookbook sre.hosts.reboot-cluster |
[production] |