|
2025-06-10
ยง
|
| 10:06 |
<jmm@cumin1003> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti2040.codfw.wmnet |
[production] |
| 10:05 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db1174 (T395241)', diff saved to https://phabricator.wikimedia.org/P77460 and previous config saved to /var/cache/conftool/dbconfig/20250610-100558-fceratto.json |
[production] |
| 10:05 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1174.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:05 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1170 (T395241)', diff saved to https://phabricator.wikimedia.org/P77459 and previous config saved to /var/cache/conftool/dbconfig/20250610-100532-fceratto.json |
[production] |
| 10:02 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2148', diff saved to https://phabricator.wikimedia.org/P77458 and previous config saved to /var/cache/conftool/dbconfig/20250610-100239-marostegui.json |
[production] |
| 10:01 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti2039.codfw.wmnet |
[production] |
| 10:00 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti2039.codfw.wmnet |
[production] |
| 09:59 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db1187 (re)pooling @ 40%: Repooling', diff saved to https://phabricator.wikimedia.org/P77457 and previous config saved to /var/cache/conftool/dbconfig/20250610-095933-root.json |
[production] |
| 09:58 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2032 (re)pooling @ 20%: Repooling', diff saved to https://phabricator.wikimedia.org/P77456 and previous config saved to /var/cache/conftool/dbconfig/20250610-095850-root.json |
[production] |
| 09:58 |
<fceratto@deploy1003> |
helmfile [aux-k8s-eqiad] 'sync' command on namespace 'zarcillo' for release 'main' . |
[production] |
| 09:55 |
<jmm@cumin1003> |
START - Cookbook sre.hosts.reboot-single for host ganeti2039.codfw.wmnet |
[production] |
| 09:55 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2033 (re)pooling @ 10%: Repooling', diff saved to https://phabricator.wikimedia.org/P77455 and previous config saved to /var/cache/conftool/dbconfig/20250610-095527-root.json |
[production] |
| 09:54 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.mysql.upgrade (exit_code=0) for es2033.codfw.wmnet |
[production] |
| 09:54 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on dbstore1009.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:50 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1170', diff saved to https://phabricator.wikimedia.org/P77454 and previous config saved to /var/cache/conftool/dbconfig/20250610-095025-fceratto.json |
[production] |
| 09:50 |
<jmm@cumin1003> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti2039.codfw.wmnet |
[production] |
| 09:49 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.ganeti.drain-node (exit_code=0) for draining ganeti node ganeti2038.codfw.wmnet |
[production] |
| 09:49 |
<jmm@cumin1003> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host ganeti2038.codfw.wmnet |
[production] |
| 09:48 |
<fnegri@cumin1003> |
conftool action : set/pooled=yes; selector: name=clouddb1019.eqiad.wmnet |
[production] |
| 09:47 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db2148 (T396130)', diff saved to https://phabricator.wikimedia.org/P77453 and previous config saved to /var/cache/conftool/dbconfig/20250610-094731-marostegui.json |
[production] |
| 09:46 |
<moritzm> |
installing postgresql-15 security updates |
[production] |
| 09:45 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.mysql.depool (exit_code=0) es2033 - Upgrading es2033.codfw.wmnet |
[production] |
| 09:45 |
<marostegui@cumin1002> |
START - Cookbook sre.mysql.depool es2033 - Upgrading es2033.codfw.wmnet |
[production] |
| 09:44 |
<marostegui@cumin1002> |
START - Cookbook sre.mysql.upgrade for es2033.codfw.wmnet |
[production] |
| 09:44 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'db1187 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P77450 and previous config saved to /var/cache/conftool/dbconfig/20250610-094429-root.json |
[production] |
| 09:44 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Promote es2031 to es2 master T395241', diff saved to https://phabricator.wikimedia.org/P77449 and previous config saved to /var/cache/conftool/dbconfig/20250610-094401-root.json |
[production] |
| 09:43 |
<jmm@cumin1003> |
START - Cookbook sre.hosts.reboot-single for host ganeti2038.codfw.wmnet |
[production] |
| 09:43 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'es2032 (re)pooling @ 10%: Repooling', diff saved to https://phabricator.wikimedia.org/P77448 and previous config saved to /var/cache/conftool/dbconfig/20250610-094345-root.json |
[production] |
| 09:43 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.mysql.upgrade (exit_code=0) for es2032.codfw.wmnet |
[production] |
| 09:39 |
<fceratto@deploy1003> |
helmfile [aux-k8s-eqiad] 'sync' command on namespace 'zarcillo' for release 'main' . |
[production] |
| 09:39 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on db1187.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:38 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depool db1187 T395989', diff saved to https://phabricator.wikimedia.org/P77447 and previous config saved to /var/cache/conftool/dbconfig/20250610-093846-marostegui.json |
[production] |
| 09:38 |
<akosiaris@deploy1003> |
helmfile [codfw] DONE helmfile.d/services/wikifunctions: apply |
[production] |
| 09:37 |
<akosiaris@deploy1003> |
helmfile [codfw] START helmfile.d/services/wikifunctions: apply |
[production] |
| 09:37 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.mysql.depool (exit_code=0) es2032 - Upgrading es2032.codfw.wmnet |
[production] |
| 09:37 |
<marostegui@cumin1002> |
START - Cookbook sre.mysql.depool es2032 - Upgrading es2032.codfw.wmnet |
[production] |
| 09:36 |
<marostegui@cumin1002> |
START - Cookbook sre.mysql.upgrade for es2032.codfw.wmnet |
[production] |
| 09:36 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Promote es2030 to es1 master T395241', diff saved to https://phabricator.wikimedia.org/P77445 and previous config saved to /var/cache/conftool/dbconfig/20250610-093628-root.json |
[production] |
| 09:36 |
<fceratto@deploy1003> |
helmfile [aux-k8s-eqiad] 'sync' command on namespace 'zarcillo' for release 'main' . |
[production] |
| 09:35 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1170', diff saved to https://phabricator.wikimedia.org/P77444 and previous config saved to /var/cache/conftool/dbconfig/20250610-093518-fceratto.json |
[production] |
| 09:33 |
<jmm@cumin1003> |
START - Cookbook sre.ganeti.drain-node for draining ganeti node ganeti2038.codfw.wmnet |
[production] |
| 09:32 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db2148 (T396130)', diff saved to https://phabricator.wikimedia.org/P77443 and previous config saved to /var/cache/conftool/dbconfig/20250610-093252-marostegui.json |
[production] |
| 09:32 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2148.codfw.wmnet with reason: Maintenance |
[production] |
| 09:31 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1230.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:31 |
<fceratto@deploy1003> |
helmfile [aux-k8s-eqiad] 'sync' command on namespace 'zarcillo' for release 'main' . |
[production] |
| 09:27 |
<fceratto@deploy1003> |
helmfile [aux-k8s-eqiad] 'sync' command on namespace 'zarcillo' for release 'main' . |
[production] |
| 09:26 |
<fnegri@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on clouddb1019.eqiad.wmnet with reason: Upgrading clouddbs T394372 |
[production] |
| 09:26 |
<jynus> |
upgrade db2197 to MariaDB 10.11 T394487 |
[production] |
| 09:24 |
<fnegri@cumin1003> |
conftool action : set/pooled=no; selector: name=clouddb1019.eqiad.wmnet |
[production] |
| 09:20 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1170 (T395241)', diff saved to https://phabricator.wikimedia.org/P77442 and previous config saved to /var/cache/conftool/dbconfig/20250610-092011-fceratto.json |
[production] |