|
2022-11-02
ยง
|
| 12:04 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1156.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:04 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1146:3312 (T318955)', diff saved to https://phabricator.wikimedia.org/P37690 and previous config saved to /var/cache/conftool/dbconfig/20221102-120436-ladsgroup.json |
[production] |
| 12:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Add some weight to es4 master', diff saved to https://phabricator.wikimedia.org/P37689 and previous config saved to /var/cache/conftool/dbconfig/20221102-120233-marostegui.json |
[production] |
| 12:04 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3316 (T318950)', diff saved to https://phabricator.wikimedia.org/P37688 and previous config saved to /var/cache/conftool/dbconfig/20221102-120209-ladsgroup.json |
[production] |
| 12:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2148 (T318955)', diff saved to https://phabricator.wikimedia.org/P37687 and previous config saved to /var/cache/conftool/dbconfig/20221102-120157-ladsgroup.json |
[production] |
| 12:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1165 (T318950)', diff saved to https://phabricator.wikimedia.org/P37686 and previous config saved to /var/cache/conftool/dbconfig/20221102-120013-ladsgroup.json |
[production] |
| 12:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2169:3316 (T318950)', diff saved to https://phabricator.wikimedia.org/P37685 and previous config saved to /var/cache/conftool/dbconfig/20221102-115948-ladsgroup.json |
[production] |
| 12:00 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2169.codfw.wmnet with reason: Maintenance |
[production] |
| 12:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2148 (T318955)', diff saved to https://phabricator.wikimedia.org/P37684 and previous config saved to /var/cache/conftool/dbconfig/20221102-115940-ladsgroup.json |
[production] |
| 12:00 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db2148.codfw.wmnet with reason: Maintenance |
[production] |
| 11:59 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2169.codfw.wmnet with reason: Maintenance |
[production] |
| 11:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2158 (T318950)', diff saved to https://phabricator.wikimedia.org/P37683 and previous config saved to /var/cache/conftool/dbconfig/20221102-115925-ladsgroup.json |
[production] |
| 11:59 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db2148.codfw.wmnet with reason: Maintenance |
[production] |
| 11:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2138:3312 (T318955)', diff saved to https://phabricator.wikimedia.org/P37682 and previous config saved to /var/cache/conftool/dbconfig/20221102-115855-ladsgroup.json |
[production] |
| 11:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1165 (T318950)', diff saved to https://phabricator.wikimedia.org/P37681 and previous config saved to /var/cache/conftool/dbconfig/20221102-115802-ladsgroup.json |
[production] |
| 11:59 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on clouddb[1015,1019,1021].eqiad.wmnet,db1155.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:59 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on clouddb[1015,1019,1021].eqiad.wmnet,db1155.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:58 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1165.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:58 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1165.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:57 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1140.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:57 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1140.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:57 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1131 (T318950)', diff saved to https://phabricator.wikimedia.org/P37680 and previous config saved to /var/cache/conftool/dbconfig/20221102-115705-ladsgroup.json |
[production] |
| 11:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool es1020', diff saved to https://phabricator.wikimedia.org/P37679 and previous config saved to /var/cache/conftool/dbconfig/20221102-115448-root.json |
[production] |
| 11:53 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Promote es1021 to es4 primary T322181', diff saved to https://phabricator.wikimedia.org/P37678 and previous config saved to /var/cache/conftool/dbconfig/20221102-115313-root.json |
[production] |
| 11:52 |
<marostegui> |
Starting es4 eqiad failover from es1020 to es1021 - T322181 |
[production] |
| 11:52 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161', diff saved to https://phabricator.wikimedia.org/P37677 and previous config saved to /var/cache/conftool/dbconfig/20221102-115233-marostegui.json |
[production] |
| 11:51 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1145.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:51 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1145.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112 (T318605)', diff saved to https://phabricator.wikimedia.org/P37676 and previous config saved to /var/cache/conftool/dbconfig/20221102-115023-ladsgroup.json |
[production] |
| 11:49 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mw-debug: apply |
[production] |
| 11:49 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1146:3312', diff saved to https://phabricator.wikimedia.org/P37675 and previous config saved to /var/cache/conftool/dbconfig/20221102-114927-ladsgroup.json |
[production] |
| 11:48 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mw-debug: apply |
[production] |
| 11:48 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-debug: apply |
[production] |
| 11:47 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-debug: apply |
[production] |
| 11:47 |
<marostegui@deploy1002> |
Finished scap: Backport for [[gerrit:852151|db-production: Disable writes one es4 (T322181)]] (duration: 04m 43s) |
[production] |
| 11:44 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2158', diff saved to https://phabricator.wikimedia.org/P37674 and previous config saved to /var/cache/conftool/dbconfig/20221102-114416-ladsgroup.json |
[production] |
| 11:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2138:3312', diff saved to https://phabricator.wikimedia.org/P37673 and previous config saved to /var/cache/conftool/dbconfig/20221102-114347-ladsgroup.json |
[production] |
| 11:43 |
<marostegui@deploy1002> |
marostegui and marostegui: Backport for [[gerrit:852151|db-production: Disable writes one es4 (T322181)]] synced to the testservers: mwdebug1002.eqiad.wmnet, mwdebug1001.eqiad.wmnet, mwdebug2001.codfw.wmnet, mwdebug2002.codfw.wmnet |
[production] |
| 11:42 |
<marostegui@deploy1002> |
Started scap: Backport for [[gerrit:852151|db-production: Disable writes one es4 (T322181)]] |
[production] |
| 11:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1131', diff saved to https://phabricator.wikimedia.org/P37672 and previous config saved to /var/cache/conftool/dbconfig/20221102-114157-ladsgroup.json |
[production] |
| 11:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Set es1021 with weight 0 T322181', diff saved to https://phabricator.wikimedia.org/P37671 and previous config saved to /var/cache/conftool/dbconfig/20221102-114107-root.json |
[production] |
| 11:41 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 6 hosts with reason: Primary switchover es4 T322181 |
[production] |
| 11:40 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 6 hosts with reason: Primary switchover es4 T322181 |
[production] |
| 11:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161', diff saved to https://phabricator.wikimedia.org/P37670 and previous config saved to /var/cache/conftool/dbconfig/20221102-113726-marostegui.json |
[production] |
| 11:37 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2109 (T318605)', diff saved to https://phabricator.wikimedia.org/P37669 and previous config saved to /var/cache/conftool/dbconfig/20221102-113542-ladsgroup.json |
[production] |
| 11:36 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2109.codfw.wmnet with reason: Maintenance |
[production] |
| 11:35 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1112', diff saved to https://phabricator.wikimedia.org/P37668 and previous config saved to /var/cache/conftool/dbconfig/20221102-113515-ladsgroup.json |
[production] |
| 11:35 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2109.codfw.wmnet with reason: Maintenance |
[production] |
| 11:35 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2105 (T318605)', diff saved to https://phabricator.wikimedia.org/P37667 and previous config saved to /var/cache/conftool/dbconfig/20221102-113506-ladsgroup.json |
[production] |
| 11:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1146:3312', diff saved to https://phabricator.wikimedia.org/P37666 and previous config saved to /var/cache/conftool/dbconfig/20221102-113419-ladsgroup.json |
[production] |