|
2022-04-21
§
|
| 03:25 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1158.eqiad.wmnet with reason: Maintenance |
[production] |
| 03:25 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1158.eqiad.wmnet with reason: Maintenance |
[production] |
| 03:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1144:3315 (T298565)', diff saved to https://phabricator.wikimedia.org/P25853 and previous config saved to /var/cache/conftool/dbconfig/20220421-032503-ladsgroup.json |
[production] |
| 03:25 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1144.eqiad.wmnet with reason: Maintenance |
[production] |
| 03:24 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1144.eqiad.wmnet with reason: Maintenance |
[production] |
| 03:13 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1179', diff saved to https://phabricator.wikimedia.org/P25852 and previous config saved to /var/cache/conftool/dbconfig/20220421-031354-ladsgroup.json |
[production] |
| 02:58 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1179 (T298565)', diff saved to https://phabricator.wikimedia.org/P25851 and previous config saved to /var/cache/conftool/dbconfig/20220421-025849-ladsgroup.json |
[production] |
| 02:39 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1130 (T298565)', diff saved to https://phabricator.wikimedia.org/P25850 and previous config saved to /var/cache/conftool/dbconfig/20220421-023942-ladsgroup.json |
[production] |
| 02:37 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1130 (T298565)', diff saved to https://phabricator.wikimedia.org/P25849 and previous config saved to /var/cache/conftool/dbconfig/20220421-023710-ladsgroup.json |
[production] |
| 02:37 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1130.eqiad.wmnet with reason: Maintenance |
[production] |
| 02:37 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1130.eqiad.wmnet with reason: Maintenance |
[production] |
| 02:32 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 02:32 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 02:31 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on 8 hosts with reason: Maintenance |
[production] |
| 02:30 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on 8 hosts with reason: Maintenance |
[production] |
| 02:30 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2123.codfw.wmnet with reason: Maintenance |
[production] |
| 02:30 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2123.codfw.wmnet with reason: Maintenance |
[production] |
| 02:26 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1150.eqiad.wmnet with reason: Maintenance |
[production] |
| 02:26 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1150.eqiad.wmnet with reason: Maintenance |
[production] |
| 02:26 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110 (T298565)', diff saved to https://phabricator.wikimedia.org/P25848 and previous config saved to /var/cache/conftool/dbconfig/20220421-022631-ladsgroup.json |
[production] |
| 02:11 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110', diff saved to https://phabricator.wikimedia.org/P25847 and previous config saved to /var/cache/conftool/dbconfig/20220421-021126-ladsgroup.json |
[production] |
| 02:07 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1179 (T298565)', diff saved to https://phabricator.wikimedia.org/P25846 and previous config saved to /var/cache/conftool/dbconfig/20220421-020727-ladsgroup.json |
[production] |
| 02:07 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1179.eqiad.wmnet with reason: Maintenance |
[production] |
| 02:07 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1179.eqiad.wmnet with reason: Maintenance |
[production] |
| 01:56 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110', diff saved to https://phabricator.wikimedia.org/P25845 and previous config saved to /var/cache/conftool/dbconfig/20220421-015621-ladsgroup.json |
[production] |
| 01:41 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1110 (T298565)', diff saved to https://phabricator.wikimedia.org/P25844 and previous config saved to /var/cache/conftool/dbconfig/20220421-014116-ladsgroup.json |
[production] |
| 01:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1110 (T298565)', diff saved to https://phabricator.wikimedia.org/P25843 and previous config saved to /var/cache/conftool/dbconfig/20220421-013456-ladsgroup.json |
[production] |
| 01:34 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1110.eqiad.wmnet with reason: Maintenance |
[production] |
| 01:34 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1110.eqiad.wmnet with reason: Maintenance |
[production] |
| 01:34 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161 (T298565)', diff saved to https://phabricator.wikimedia.org/P25842 and previous config saved to /var/cache/conftool/dbconfig/20220421-013401-ladsgroup.json |
[production] |
| 01:22 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1145.eqiad.wmnet with reason: Maintenance |
[production] |
| 01:22 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1145.eqiad.wmnet with reason: Maintenance |
[production] |
| 01:22 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175 (T298565)', diff saved to https://phabricator.wikimedia.org/P25841 and previous config saved to /var/cache/conftool/dbconfig/20220421-012235-ladsgroup.json |
[production] |
| 01:18 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161', diff saved to https://phabricator.wikimedia.org/P25840 and previous config saved to /var/cache/conftool/dbconfig/20220421-011856-ladsgroup.json |
[production] |
| 01:07 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175', diff saved to https://phabricator.wikimedia.org/P25839 and previous config saved to /var/cache/conftool/dbconfig/20220421-010730-ladsgroup.json |
[production] |
| 01:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161', diff saved to https://phabricator.wikimedia.org/P25838 and previous config saved to /var/cache/conftool/dbconfig/20220421-010351-ladsgroup.json |
[production] |
| 00:52 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175', diff saved to https://phabricator.wikimedia.org/P25837 and previous config saved to /var/cache/conftool/dbconfig/20220421-005225-ladsgroup.json |
[production] |
| 00:48 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1161 (T298565)', diff saved to https://phabricator.wikimedia.org/P25836 and previous config saved to /var/cache/conftool/dbconfig/20220421-004846-ladsgroup.json |
[production] |
| 00:37 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1175 (T298565)', diff saved to https://phabricator.wikimedia.org/P25835 and previous config saved to /var/cache/conftool/dbconfig/20220421-003720-ladsgroup.json |
[production] |
| 00:30 |
<mutante> |
alert1001 - sudo systemctl start certspotter - another time, not on our end but should probably fail more gracefully |
[production] |
| 00:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1175 (T298565)', diff saved to https://phabricator.wikimedia.org/P25834 and previous config saved to /var/cache/conftool/dbconfig/20220421-002107-ladsgroup.json |
[production] |
| 00:21 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1175.eqiad.wmnet with reason: Maintenance |
[production] |
| 00:21 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1175.eqiad.wmnet with reason: Maintenance |
[production] |
| 00:09 |
<mutante> |
alert1001 - sudo systemctl start certspotter (after an alert from Icinga itself that it failed. error was some temp error fetching data from comodo) |
[production] |