|
2022-01-31
ยง
|
| 11:23 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: sync on pinkunicorn |
[production] |
| 11:22 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply on pinkunicorn |
[production] |
| 11:21 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: sync on pinkunicorn |
[production] |
| 11:21 |
<hnowlan@cumin1001> |
START - Cookbook sre.hosts.reimage for host restbase1025.eqiad.wmnet with OS buster |
[production] |
| 11:20 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply on pinkunicorn |
[production] |
| 11:19 |
<hnowlan@puppetmaster1001> |
conftool action : set/pooled=yes; selector: name=restbase1024.eqiad.wmnet |
[production] |
| 11:19 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175 (T298559)', diff saved to https://phabricator.wikimedia.org/P19614 and previous config saved to /var/cache/conftool/dbconfig/20220131-111904-marostegui.json |
[production] |
| 11:11 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1096:3315 (re)pooling @ 100%: repooling', diff saved to https://phabricator.wikimedia.org/P19613 and previous config saved to /var/cache/conftool/dbconfig/20220131-111147-root.json |
[production] |
| 11:08 |
<hnowlan@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 15 days, 0:00:00 on restbase2009.codfw.wmnet with reason: not in restbase cluster, used for testing |
[production] |
| 11:08 |
<hnowlan@cumin1001> |
START - Cookbook sre.hosts.downtime for 15 days, 0:00:00 on restbase2009.codfw.wmnet with reason: not in restbase cluster, used for testing |
[production] |
| 11:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175', diff saved to https://phabricator.wikimedia.org/P19612 and previous config saved to /var/cache/conftool/dbconfig/20220131-110400-marostegui.json |
[production] |
| 10:58 |
<vgutierrez> |
pool cp5011 running envoy as TLS terminator - T271421 |
[production] |
| 10:57 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cp5011.eqsin.wmnet with OS buster |
[production] |
| 10:56 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1096:3315 (re)pooling @ 75%: repooling', diff saved to https://phabricator.wikimedia.org/P19611 and previous config saved to /var/cache/conftool/dbconfig/20220131-105643-root.json |
[production] |
| 10:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175', diff saved to https://phabricator.wikimedia.org/P19610 and previous config saved to /var/cache/conftool/dbconfig/20220131-104855-marostegui.json |
[production] |
| 10:42 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ganeti1015.eqiad.wmnet with OS buster |
[production] |
| 10:41 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1096:3315 (re)pooling @ 50%: repooling', diff saved to https://phabricator.wikimedia.org/P19609 and previous config saved to /var/cache/conftool/dbconfig/20220131-104140-root.json |
[production] |
| 10:37 |
<mmandere> |
cp[1087,1089-1090] remove unused libvarnishapi1 T300247 |
[production] |
| 10:36 |
<mmandere> |
cp[2041-2042] remove unused libvarnishapi1 T300247 |
[production] |
| 10:33 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175 (T298559)', diff saved to https://phabricator.wikimedia.org/P19608 and previous config saved to /var/cache/conftool/dbconfig/20220131-103350-marostegui.json |
[production] |
| 10:33 |
<mmandere> |
cp[3052,3064-3065].esams.wmnet remove unused libvarnishapi1 T300247 |
[production] |
| 10:31 |
<mmandere> |
cp[4021,4025-4026,4032-4034,4036].ulsfo.wmnet remove unused libvarnishapi1 T300247 |
[production] |
| 10:27 |
<mmandere> |
cp[5006,5012].eqsin.wmnet remove unused libvarnishapi1 T300247 |
[production] |
| 10:26 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1096:3315 (re)pooling @ 25%: repooling', diff saved to https://phabricator.wikimedia.org/P19607 and previous config saved to /var/cache/conftool/dbconfig/20220131-102636-root.json |
[production] |
| 10:24 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1096:3315', diff saved to https://phabricator.wikimedia.org/P19606 and previous config saved to /var/cache/conftool/dbconfig/20220131-102457-marostegui.json |
[production] |
| 10:21 |
<moritzm> |
installing apache/apache-modsecurity2 security updates |
[production] |
| 10:15 |
<mmandere> |
cp[6001-6016].drmrs.wmnet remove unused libvarnishapi1 T300247 |
[production] |
| 10:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1175 (T298559)', diff saved to https://phabricator.wikimedia.org/P19605 and previous config saved to /var/cache/conftool/dbconfig/20220131-101439-marostegui.json |
[production] |
| 10:14 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1175.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:14 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1175.eqiad.wmnet with reason: Maintenance |
[production] |
| 10:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1179 (T298559)', diff saved to https://phabricator.wikimedia.org/P19604 and previous config saved to /var/cache/conftool/dbconfig/20220131-101431-marostegui.json |
[production] |
| 10:01 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reimage for host ganeti1015.eqiad.wmnet with OS buster |
[production] |
| 09:59 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1179', diff saved to https://phabricator.wikimedia.org/P19603 and previous config saved to /var/cache/conftool/dbconfig/20220131-095926-marostegui.json |
[production] |
| 09:55 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.reimage for host cp5011.eqsin.wmnet with OS buster |
[production] |
| 09:53 |
<mmandere> |
cp3062: upgrade varnish to 6.0.10-1wm1 T300264 |
[production] |
| 09:53 |
<vgutierrez> |
depool cp5011 to be reimaged as cache::text_envoy - T271421 |
[production] |
| 09:48 |
<hnowlan@cumin1001> |
START - Cookbook sre.postgresql.postgres-init |
[production] |
| 09:48 |
<mmandere> |
cp3061: upgrade varnish to 6.0.10-1wm1 T300264 |
[production] |
| 09:45 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ganeti1007.eqiad.wmnet with OS buster |
[production] |
| 09:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1179', diff saved to https://phabricator.wikimedia.org/P19602 and previous config saved to /var/cache/conftool/dbconfig/20220131-094422-marostegui.json |
[production] |
| 09:35 |
<dcausse> |
restart blazegraph on wdqs1012 (jvm stuck for 6hours) |
[production] |
| 09:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1096:3315 (re)pooling @ 100%: repooling after reimage', diff saved to https://phabricator.wikimedia.org/P19601 and previous config saved to /var/cache/conftool/dbconfig/20220131-093450-root.json |
[production] |
| 09:29 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1179 (T298559)', diff saved to https://phabricator.wikimedia.org/P19600 and previous config saved to /var/cache/conftool/dbconfig/20220131-092917-marostegui.json |
[production] |
| 09:21 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reimage for host ganeti1007.eqiad.wmnet with OS buster |
[production] |
| 09:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1179 (T298559)', diff saved to https://phabricator.wikimedia.org/P19599 and previous config saved to /var/cache/conftool/dbconfig/20220131-091959-marostegui.json |
[production] |
| 09:19 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1179.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:19 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1179.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:19 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1166 (T298559)', diff saved to https://phabricator.wikimedia.org/P19598 and previous config saved to /var/cache/conftool/dbconfig/20220131-091952-marostegui.json |
[production] |
| 09:19 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1096:3315 (re)pooling @ 75%: repooling after reimage', diff saved to https://phabricator.wikimedia.org/P19597 and previous config saved to /var/cache/conftool/dbconfig/20220131-091943-root.json |
[production] |
| 09:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1166', diff saved to https://phabricator.wikimedia.org/P19596 and previous config saved to /var/cache/conftool/dbconfig/20220131-090441-marostegui.json |
[production] |