|
2022-11-24
ยง
|
| 07:09 |
<oblivian@deploy1002> |
helmfile [codfw] START helmfile.d/services/citoid: apply |
[production] |
| 07:08 |
<oblivian@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/citoid: apply |
[production] |
| 07:07 |
<oblivian@deploy1002> |
helmfile [eqiad] START helmfile.d/services/citoid: apply |
[production] |
| 07:06 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:06 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:06 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1099:3318 (T321126)', diff saved to https://phabricator.wikimedia.org/P40936 and previous config saved to /var/cache/conftool/dbconfig/20221124-070645-marostegui.json |
[production] |
| 07:06 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1187', diff saved to https://phabricator.wikimedia.org/P40935 and previous config saved to /var/cache/conftool/dbconfig/20221124-070603-ladsgroup.json |
[production] |
| 07:05 |
<oblivian@deploy1002> |
helmfile [eqiad] START helmfile.d/services/citoid: apply |
[production] |
| 07:05 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depool db1181 T323117', diff saved to https://phabricator.wikimedia.org/P40934 and previous config saved to /var/cache/conftool/dbconfig/20221124-070546-ladsgroup.json |
[production] |
| 07:05 |
<oblivian@deploy1002> |
helmfile [staging] DONE helmfile.d/services/citoid: apply |
[production] |
| 07:05 |
<oblivian@deploy1002> |
helmfile [staging] START helmfile.d/services/citoid: apply |
[production] |
| 07:04 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1099:3318 (T321126)', diff saved to https://phabricator.wikimedia.org/P40933 and previous config saved to /var/cache/conftool/dbconfig/20221124-070437-marostegui.json |
[production] |
| 07:04 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1099.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:04 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1099.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:03 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:03 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1136.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Promote db1136 to s7 primary and set section read-write T323117', diff saved to https://phabricator.wikimedia.org/P40932 and previous config saved to /var/cache/conftool/dbconfig/20221124-070250-ladsgroup.json |
[production] |
| 07:02 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:02 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:02 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set s7 eqiad as read-only for maintenance - T323117', diff saved to https://phabricator.wikimedia.org/P40931 and previous config saved to /var/cache/conftool/dbconfig/20221124-070215-ladsgroup.json |
[production] |
| 07:02 |
<Amir1> |
Starting s7 eqiad failover from db1181 to db1136 - T323117 |
[production] |
| 06:59 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40930 and previous config saved to /var/cache/conftool/dbconfig/20221124-065956-ladsgroup.json |
[production] |
| 06:56 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db2118.codfw.wmnet with reason: Maintenance |
[production] |
| 06:56 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db2118.codfw.wmnet with reason: Maintenance |
[production] |
| 06:50 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1187 (T323214)', diff saved to https://phabricator.wikimedia.org/P40929 and previous config saved to /var/cache/conftool/dbconfig/20221124-065057-ladsgroup.json |
[production] |
| 06:07 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set db1136 with weight 0 T323117', diff saved to https://phabricator.wikimedia.org/P40928 and previous config saved to /var/cache/conftool/dbconfig/20221124-060742-ladsgroup.json |
[production] |
| 06:06 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 30 hosts with reason: Primary switchover s7 T323117 |
[production] |
| 06:06 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 30 hosts with reason: Primary switchover s7 T323117 |
[production] |
| 06:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1187 (T323214)', diff saved to https://phabricator.wikimedia.org/P40927 and previous config saved to /var/cache/conftool/dbconfig/20221124-060330-ladsgroup.json |
[production] |
| 06:03 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1187.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:03 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1187.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40926 and previous config saved to /var/cache/conftool/dbconfig/20221124-060309-ladsgroup.json |
[production] |
| 05:48 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P40925 and previous config saved to /var/cache/conftool/dbconfig/20221124-054802-ladsgroup.json |
[production] |
| 05:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P40924 and previous config saved to /var/cache/conftool/dbconfig/20221124-053256-ladsgroup.json |
[production] |
| 05:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40923 and previous config saved to /var/cache/conftool/dbconfig/20221124-052830-ladsgroup.json |
[production] |
| 05:28 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2180.codfw.wmnet with reason: Maintenance |
[production] |
| 05:28 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2180.codfw.wmnet with reason: Maintenance |
[production] |
| 05:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2171:3316 (T323214)', diff saved to https://phabricator.wikimedia.org/P40922 and previous config saved to /var/cache/conftool/dbconfig/20221124-052808-ladsgroup.json |
[production] |
| 05:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40921 and previous config saved to /var/cache/conftool/dbconfig/20221124-051749-ladsgroup.json |
[production] |
| 05:13 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2171:3316', diff saved to https://phabricator.wikimedia.org/P40920 and previous config saved to /var/cache/conftool/dbconfig/20221124-051301-ladsgroup.json |
[production] |
| 04:57 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2171:3316', diff saved to https://phabricator.wikimedia.org/P40919 and previous config saved to /var/cache/conftool/dbconfig/20221124-045755-ladsgroup.json |
[production] |
| 04:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2171:3316 (T323214)', diff saved to https://phabricator.wikimedia.org/P40918 and previous config saved to /var/cache/conftool/dbconfig/20221124-044249-ladsgroup.json |
[production] |
| 04:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1180 (T323214)', diff saved to https://phabricator.wikimedia.org/P40917 and previous config saved to /var/cache/conftool/dbconfig/20221124-042757-ladsgroup.json |
[production] |
| 04:27 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1180.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:27 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1180.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T323214)', diff saved to https://phabricator.wikimedia.org/P40916 and previous config saved to /var/cache/conftool/dbconfig/20221124-042736-ladsgroup.json |
[production] |
| 04:12 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P40915 and previous config saved to /var/cache/conftool/dbconfig/20221124-041230-ladsgroup.json |
[production] |
| 03:57 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P40914 and previous config saved to /var/cache/conftool/dbconfig/20221124-035723-ladsgroup.json |
[production] |
| 03:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T323214)', diff saved to https://phabricator.wikimedia.org/P40913 and previous config saved to /var/cache/conftool/dbconfig/20221124-034217-ladsgroup.json |
[production] |
| 03:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2171:3316 (T323214)', diff saved to https://phabricator.wikimedia.org/P40912 and previous config saved to /var/cache/conftool/dbconfig/20221124-030901-ladsgroup.json |
[production] |