|
2025-06-11
ยง
|
| 11:13 |
<jmm@cumin1003> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1043.eqiad.wmnet |
[production] |
| 11:09 |
<taavi@cloudcumin1001> |
END (FAIL) - Cookbook wmcs.toolforge.k8s.logging.copy_images_to_registry (exit_code=99) for Loki 3.5.0, Alloy 1.9.1 |
[tools] |
| 11:09 |
<taavi@cloudcumin1001> |
Updating container image docker-registry.svc.toolforge.org/grafana/loki:3.5.0 |
[tools] |
| 11:09 |
<taavi@cloudcumin1001> |
START - Cookbook wmcs.toolforge.k8s.logging.copy_images_to_registry for Loki 3.5.0, Alloy 1.9.1 |
[tools] |
| 11:06 |
<mvolz@deploy1003> |
helmfile [eqiad] DONE helmfile.d/services/citoid: apply |
[production] |
| 11:06 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1042.eqiad.wmnet |
[production] |
| 11:06 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1042.eqiad.wmnet |
[production] |
| 11:06 |
<mvolz@deploy1003> |
helmfile [eqiad] START helmfile.d/services/citoid: apply |
[production] |
| 11:05 |
<mvolz@deploy1003> |
helmfile [codfw] DONE helmfile.d/services/citoid: apply |
[production] |
| 11:05 |
<mvolz@deploy1003> |
helmfile [codfw] START helmfile.d/services/citoid: apply |
[production] |
| 11:03 |
<mvolz@deploy1003> |
helmfile [staging] DONE helmfile.d/services/citoid: apply |
[production] |
| 11:02 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2205', diff saved to https://phabricator.wikimedia.org/P77697 and previous config saved to /var/cache/conftool/dbconfig/20250611-110257-marostegui.json |
[production] |
| 11:02 |
<mvolz@deploy1003> |
helmfile [staging] START helmfile.d/services/citoid: apply |
[production] |
| 11:02 |
<brouberol@deploy1003> |
helmfile [eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 11:02 |
<brouberol@deploy1003> |
helmfile [eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 11:01 |
<jmm@cumin1003> |
START - Cookbook sre.hosts.reboot-single for host ganeti1042.eqiad.wmnet |
[production] |
| 10:59 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1196 (T395241)', diff saved to https://phabricator.wikimedia.org/P77696 and previous config saved to /var/cache/conftool/dbconfig/20250611-105900-fceratto.json |
[production] |
| 10:57 |
<jmm@cumin1003> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1042.eqiad.wmnet |
[production] |
| 10:55 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1041.eqiad.wmnet |
[production] |
| 10:55 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1041.eqiad.wmnet |
[production] |
| 10:50 |
<jmm@cumin1003> |
START - Cookbook sre.hosts.reboot-single for host ganeti1041.eqiad.wmnet |
[production] |
| 10:48 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db1196 (T395241)', diff saved to https://phabricator.wikimedia.org/P77695 and previous config saved to /var/cache/conftool/dbconfig/20250611-104825-fceratto.json |
[production] |
| 10:48 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on an-redacteddb1001.eqiad.wmnet,clouddb[1013,1017].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:48 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1196.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:47 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2205 (T396130)', diff saved to https://phabricator.wikimedia.org/P77694 and previous config saved to /var/cache/conftool/dbconfig/20250611-104750-marostegui.json |
[production] |
| 10:47 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1195 (T395241)', diff saved to https://phabricator.wikimedia.org/P77693 and previous config saved to /var/cache/conftool/dbconfig/20250611-104741-fceratto.json |
[production] |
| 10:46 |
<jmm@cumin1003> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1041.eqiad.wmnet |
[production] |
| 10:45 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1040.eqiad.wmnet |
[production] |
| 10:45 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1040.eqiad.wmnet |
[production] |
| 10:40 |
<jmm@cumin1003> |
START - Cookbook sre.hosts.reboot-single for host ganeti1040.eqiad.wmnet |
[production] |
| 10:32 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1195', diff saved to https://phabricator.wikimedia.org/P77692 and previous config saved to /var/cache/conftool/dbconfig/20250611-103234-fceratto.json |
[production] |
| 10:32 |
<jmm@cumin1003> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1040.eqiad.wmnet |
[production] |
| 10:32 |
<fceratto@deploy1003> |
helmfile [aux-k8s-eqiad] 'sync' command on namespace 'zarcillo' for release 'main' . |
[production] |
| 10:29 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1039.eqiad.wmnet |
[production] |
| 10:29 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1039.eqiad.wmnet |
[production] |
| 10:29 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2205 (T396130)', diff saved to https://phabricator.wikimedia.org/P77691 and previous config saved to /var/cache/conftool/dbconfig/20250611-102902-marostegui.json |
[production] |
| 10:28 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2205.codfw.wmnet with reason: Maintenance |
[production] |
| 10:28 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2194 (T396130)', diff saved to https://phabricator.wikimedia.org/P77690 and previous config saved to /var/cache/conftool/dbconfig/20250611-102839-marostegui.json |
[production] |
| 10:23 |
<jmm@cumin1003> |
START - Cookbook sre.hosts.reboot-single for host ganeti1039.eqiad.wmnet |
[production] |
| 10:20 |
<jmm@cumin1003> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1039.eqiad.wmnet |
[production] |
| 10:17 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1195', diff saved to https://phabricator.wikimedia.org/P77689 and previous config saved to /var/cache/conftool/dbconfig/20250611-101727-fceratto.json |
[production] |
| 10:15 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti1038.eqiad.wmnet |
[production] |
| 10:15 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti1038.eqiad.wmnet |
[production] |
| 10:13 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2194', diff saved to https://phabricator.wikimedia.org/P77688 and previous config saved to /var/cache/conftool/dbconfig/20250611-101332-marostegui.json |
[production] |
| 10:09 |
<fceratto@deploy1003> |
helmfile [aux-k8s-eqiad] 'sync' command on namespace 'zarcillo' for release 'main' . |
[production] |
| 10:07 |
<jmm@cumin1003> |
START - Cookbook sre.hosts.reboot-single for host ganeti1038.eqiad.wmnet |
[production] |
| 10:06 |
<jmm@cumin1003> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti1038.eqiad.wmnet |
[production] |
| 10:02 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1195 (T395241)', diff saved to https://phabricator.wikimedia.org/P77687 and previous config saved to /var/cache/conftool/dbconfig/20250611-100220-fceratto.json |
[production] |
| 10:00 |
<brouberol@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host dse-k8s-worker1013.eqiad.wmnet with OS bookworm |
[production] |
| 09:58 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2194', diff saved to https://phabricator.wikimedia.org/P77686 and previous config saved to /var/cache/conftool/dbconfig/20250611-095825-marostegui.json |
[production] |