|
2022-01-27
ยง
|
| 09:23 |
<root@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on db[2078,2132].codfw.wmnet,db[1117,1128,1159].eqiad.wmnet with reason: Primary switchover m1 T299624 |
[production] |
| 09:22 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1131 (re)pooling @ 50%: repooling after reimage', diff saved to https://phabricator.wikimedia.org/P19409 and previous config saved to /var/cache/conftool/dbconfig/20220127-092251-root.json |
[production] |
| 09:18 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4 days, 0:00:00 on ganeti1007.eqiad.wmnet with reason: Remove from Ganeti cluster for reimage |
[production] |
| 09:18 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.downtime for 4 days, 0:00:00 on ganeti1007.eqiad.wmnet with reason: Remove from Ganeti cluster for reimage |
[production] |
| 09:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112 (T285149)', diff saved to https://phabricator.wikimedia.org/P19408 and previous config saved to /var/cache/conftool/dbconfig/20220127-091641-marostegui.json |
[production] |
| 09:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3317 (T298559)', diff saved to https://phabricator.wikimedia.org/P19407 and previous config saved to /var/cache/conftool/dbconfig/20220127-091453-marostegui.json |
[production] |
| 09:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1170:3317 (T298559)', diff saved to https://phabricator.wikimedia.org/P19406 and previous config saved to /var/cache/conftool/dbconfig/20220127-091440-marostegui.json |
[production] |
| 09:14 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:14 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:14 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on 10 hosts with reason: Maintenance |
[production] |
| 09:14 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on 10 hosts with reason: Maintenance |
[production] |
| 09:14 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2121.codfw.wmnet with reason: Maintenance |
[production] |
| 09:14 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2121.codfw.wmnet with reason: Maintenance |
[production] |
| 09:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127 (T298559)', diff saved to https://phabricator.wikimedia.org/P19405 and previous config saved to /var/cache/conftool/dbconfig/20220127-091401-marostegui.json |
[production] |
| 09:07 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1131 (re)pooling @ 40%: repooling after reimage', diff saved to https://phabricator.wikimedia.org/P19404 and previous config saved to /var/cache/conftool/dbconfig/20220127-090747-root.json |
[production] |
| 08:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127', diff saved to https://phabricator.wikimedia.org/P19403 and previous config saved to /var/cache/conftool/dbconfig/20220127-085857-marostegui.json |
[production] |
| 08:52 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1131 (re)pooling @ 25%: repooling after reimage', diff saved to https://phabricator.wikimedia.org/P19402 and previous config saved to /var/cache/conftool/dbconfig/20220127-085244-root.json |
[production] |
| 08:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127', diff saved to https://phabricator.wikimedia.org/P19401 and previous config saved to /var/cache/conftool/dbconfig/20220127-084352-marostegui.json |
[production] |
| 08:41 |
<jayme@deploy1002> |
Finished deploy [restbase/deploy@0848b15]: scap testing (duration: 00m 05s) |
[production] |
| 08:40 |
<jayme@deploy1002> |
Started deploy [restbase/deploy@0848b15]: scap testing |
[production] |
| 08:38 |
<jayme> |
updated scap to 4.2.1 on A:mw-canary, A:parsoid-canary, A:mw-jobrunner-canary, A:restbase-canary - T300058 |
[production] |
| 08:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1131 (re)pooling @ 20%: repooling after reimage', diff saved to https://phabricator.wikimedia.org/P19400 and previous config saved to /var/cache/conftool/dbconfig/20220127-083740-root.json |
[production] |
| 08:33 |
<jayme> |
uploaded scap 4.2.1 to apt.wikimedia.org - T300058 |
[production] |
| 08:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1127 (T298559)', diff saved to https://phabricator.wikimedia.org/P19399 and previous config saved to /var/cache/conftool/dbconfig/20220127-082847-marostegui.json |
[production] |
| 08:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1127 (T298559)', diff saved to https://phabricator.wikimedia.org/P19398 and previous config saved to /var/cache/conftool/dbconfig/20220127-082735-marostegui.json |
[production] |
| 08:27 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1127.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:27 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1127.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317 (T298559)', diff saved to https://phabricator.wikimedia.org/P19397 and previous config saved to /var/cache/conftool/dbconfig/20220127-082728-marostegui.json |
[production] |
| 08:22 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1131 (re)pooling @ 10%: repooling after reimage', diff saved to https://phabricator.wikimedia.org/P19396 and previous config saved to /var/cache/conftool/dbconfig/20220127-082236-root.json |
[production] |
| 08:21 |
<jayme@deploy1002> |
helmfile [staging-codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 08:21 |
<jayme@deploy1002> |
helmfile [staging-codfw] START helmfile.d/admin 'apply'. |
[production] |
| 08:19 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: sync on pinkunicorn |
[production] |
| 08:17 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply on pinkunicorn |
[production] |
| 08:17 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: sync on pinkunicorn |
[production] |
| 08:16 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply on pinkunicorn |
[production] |
| 08:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1112 (T285149)', diff saved to https://phabricator.wikimedia.org/P19395 and previous config saved to /var/cache/conftool/dbconfig/20220127-081622-marostegui.json |
[production] |
| 08:16 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:16 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on clouddb[1013,1017,1021].eqiad.wmnet,db1154.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:16 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1112.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:16 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1112.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:13 |
<ladsgroup@deploy1002> |
Synchronized php-1.38.0-wmf.19/includes/libs/rdbms/database/Database.php: Backport: [[gerrit:757476|Don't consider lock waits to be write queries (T300194)]] (duration: 00m 52s) |
[production] |
| 08:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P19394 and previous config saved to /var/cache/conftool/dbconfig/20220127-081223-marostegui.json |
[production] |
| 08:07 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:07 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on dbstore1007.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:07 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1131 (re)pooling @ 5%: repooling after reimage', diff saved to https://phabricator.wikimedia.org/P19393 and previous config saved to /var/cache/conftool/dbconfig/20220127-080733-root.json |
[production] |
| 07:59 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:59 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:59 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1123 (T285149)', diff saved to https://phabricator.wikimedia.org/P19392 and previous config saved to /var/cache/conftool/dbconfig/20220127-075909-marostegui.json |
[production] |
| 07:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P19391 and previous config saved to /var/cache/conftool/dbconfig/20220127-075718-marostegui.json |
[production] |
| 07:52 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1131 (re)pooling @ 1%: repooling after reimage', diff saved to https://phabricator.wikimedia.org/P19390 and previous config saved to /var/cache/conftool/dbconfig/20220127-075229-root.json |
[production] |