|
2022-03-31
ยง
|
| 07:16 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 06:49 |
<elukey@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ml-cache1002.eqiad.wmnet with OS bullseye |
[production] |
| 06:48 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.reimage for host ml-cache1002.eqiad.wmnet with OS bullseye |
[production] |
| 06:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1129 (T298565)', diff saved to https://phabricator.wikimedia.org/P23993 and previous config saved to /var/cache/conftool/dbconfig/20220331-063429-ladsgroup.json |
[production] |
| 06:19 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1129', diff saved to https://phabricator.wikimedia.org/P23992 and previous config saved to /var/cache/conftool/dbconfig/20220331-061923-ladsgroup.json |
[production] |
| 06:12 |
<marostegui> |
dbmaint s5@eqiad T300381 |
[production] |
| 06:08 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1130 T303798', diff saved to https://phabricator.wikimedia.org/P23991 and previous config saved to /var/cache/conftool/dbconfig/20220331-060820-root.json |
[production] |
| 06:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1149 (T298557)', diff saved to https://phabricator.wikimedia.org/P23990 and previous config saved to /var/cache/conftool/dbconfig/20220331-060517-marostegui.json |
[production] |
| 06:05 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1149.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:05 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1149.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148 (T298557)', diff saved to https://phabricator.wikimedia.org/P23989 and previous config saved to /var/cache/conftool/dbconfig/20220331-060509-marostegui.json |
[production] |
| 06:04 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1129', diff saved to https://phabricator.wikimedia.org/P23988 and previous config saved to /var/cache/conftool/dbconfig/20220331-060418-ladsgroup.json |
[production] |
| 06:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Promote db1100 to s5 primary and set section read-write T303798', diff saved to https://phabricator.wikimedia.org/P23987 and previous config saved to /var/cache/conftool/dbconfig/20220331-060122-root.json |
[production] |
| 06:00 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Set s5 eqiad as read-only for maintenance - T303798', diff saved to https://phabricator.wikimedia.org/P23986 and previous config saved to /var/cache/conftool/dbconfig/20220331-060042-root.json |
[production] |
| 06:00 |
<marostegui> |
Starting s5 eqiad failover from db1130 to db1100 - T303798 |
[production] |
| 05:50 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148', diff saved to https://phabricator.wikimedia.org/P23985 and previous config saved to /var/cache/conftool/dbconfig/20220331-055004-marostegui.json |
[production] |
| 05:49 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1129 (T298565)', diff saved to https://phabricator.wikimedia.org/P23984 and previous config saved to /var/cache/conftool/dbconfig/20220331-054913-ladsgroup.json |
[production] |
| 05:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148', diff saved to https://phabricator.wikimedia.org/P23983 and previous config saved to /var/cache/conftool/dbconfig/20220331-053459-marostegui.json |
[production] |
| 05:19 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1148 (T298557)', diff saved to https://phabricator.wikimedia.org/P23981 and previous config saved to /var/cache/conftool/dbconfig/20220331-051954-marostegui.json |
[production] |
| 04:49 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1129 (T298565)', diff saved to https://phabricator.wikimedia.org/P23980 and previous config saved to /var/cache/conftool/dbconfig/20220331-044859-ladsgroup.json |
[production] |
| 04:49 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1129.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:48 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1129.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:48 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182 (T298565)', diff saved to https://phabricator.wikimedia.org/P23979 and previous config saved to /var/cache/conftool/dbconfig/20220331-044851-ladsgroup.json |
[production] |
| 04:39 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Set db1100 with weight 0 T303798', diff saved to https://phabricator.wikimedia.org/P23978 and previous config saved to /var/cache/conftool/dbconfig/20220331-043906-marostegui.json |
[production] |
| 04:38 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on 22 hosts with reason: Primary switchover s5 T303798 |
[production] |
| 04:38 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on 22 hosts with reason: Primary switchover s5 T303798 |
[production] |
| 04:33 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182', diff saved to https://phabricator.wikimedia.org/P23977 and previous config saved to /var/cache/conftool/dbconfig/20220331-043346-ladsgroup.json |
[production] |
| 04:18 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182', diff saved to https://phabricator.wikimedia.org/P23976 and previous config saved to /var/cache/conftool/dbconfig/20220331-041841-ladsgroup.json |
[production] |
| 04:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1181 (T298565)', diff saved to https://phabricator.wikimedia.org/P23975 and previous config saved to /var/cache/conftool/dbconfig/20220331-040940-ladsgroup.json |
[production] |
| 04:09 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:09 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 04:09 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1181 (T298565)', diff saved to https://phabricator.wikimedia.org/P23974 and previous config saved to /var/cache/conftool/dbconfig/20220331-040916-ladsgroup.json |
[production] |
| 04:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1182 (T298565)', diff saved to https://phabricator.wikimedia.org/P23973 and previous config saved to /var/cache/conftool/dbconfig/20220331-040336-ladsgroup.json |
[production] |
| 03:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1181', diff saved to https://phabricator.wikimedia.org/P23972 and previous config saved to /var/cache/conftool/dbconfig/20220331-035411-ladsgroup.json |
[production] |
| 03:47 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1148 (T298557)', diff saved to https://phabricator.wikimedia.org/P23971 and previous config saved to /var/cache/conftool/dbconfig/20220331-034709-marostegui.json |
[production] |
| 03:47 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 8:00:00 on db1148.eqiad.wmnet with reason: Maintenance |
[production] |
| 03:47 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 8:00:00 on db1148.eqiad.wmnet with reason: Maintenance |
[production] |
| 03:47 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1147 (T298557)', diff saved to https://phabricator.wikimedia.org/P23970 and previous config saved to /var/cache/conftool/dbconfig/20220331-034701-marostegui.json |
[production] |
| 03:39 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1181', diff saved to https://phabricator.wikimedia.org/P23969 and previous config saved to /var/cache/conftool/dbconfig/20220331-033906-ladsgroup.json |
[production] |
| 03:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1147', diff saved to https://phabricator.wikimedia.org/P23968 and previous config saved to /var/cache/conftool/dbconfig/20220331-033156-marostegui.json |
[production] |
| 03:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1181 (T298565)', diff saved to https://phabricator.wikimedia.org/P23967 and previous config saved to /var/cache/conftool/dbconfig/20220331-032401-ladsgroup.json |
[production] |
| 03:16 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1147', diff saved to https://phabricator.wikimedia.org/P23966 and previous config saved to /var/cache/conftool/dbconfig/20220331-031651-marostegui.json |
[production] |
| 03:15 |
<ejegg> |
civicrm revision changed from a6f49bb3 to 84c737b6 |
[production] |
| 03:05 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1181 (T298565)', diff saved to https://phabricator.wikimedia.org/P23965 and previous config saved to /var/cache/conftool/dbconfig/20220331-030531-ladsgroup.json |
[production] |
| 03:05 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 03:05 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1181.eqiad.wmnet with reason: Maintenance |
[production] |
| 03:05 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1174 (T298565)', diff saved to https://phabricator.wikimedia.org/P23964 and previous config saved to /var/cache/conftool/dbconfig/20220331-030523-ladsgroup.json |
[production] |
| 03:04 |
<eileen> |
civicrm revision changed from a9c323af to a6f49bb3 |
[production] |
| 03:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1182 (T298565)', diff saved to https://phabricator.wikimedia.org/P23963 and previous config saved to /var/cache/conftool/dbconfig/20220331-030321-ladsgroup.json |
[production] |
| 03:03 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1182.eqiad.wmnet with reason: Maintenance |
[production] |