|
2024-07-08
§
|
| 05:59 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 28008 |
[production] |
| 05:39 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 17072 |
[production] |
| 05:39 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 17072 |
[production] |
| 05:39 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 263522 |
[production] |
| 05:38 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 263522 |
[production] |
| 05:38 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 61942 |
[production] |
| 05:38 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 61942 |
[production] |
| 05:38 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 18013 |
[production] |
| 05:37 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 18013 |
[production] |
| 05:37 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 268248 |
[production] |
| 05:37 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 268248 |
[production] |
| 05:36 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 61672 |
[production] |
| 05:36 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 61672 |
[production] |
| 05:36 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 28352 |
[production] |
| 05:36 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 28352 |
[production] |
| 05:36 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 999 |
[production] |
| 05:36 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 999 |
[production] |
| 05:35 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 4788 |
[production] |
| 05:34 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 4788 |
[production] |
| 05:34 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 132167 |
[production] |
| 05:33 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 132167 |
[production] |
| 05:33 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 6447 |
[production] |
| 05:32 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 6447 |
[production] |
| 05:31 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1210 (T367856)', diff saved to https://phabricator.wikimedia.org/P65926 and previous config saved to /var/cache/conftool/dbconfig/20240708-053133-marostegui.json |
[production] |
| 05:31 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1210.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:31 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1210.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:31 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1200 (T367856)', diff saved to https://phabricator.wikimedia.org/P65925 and previous config saved to /var/cache/conftool/dbconfig/20240708-053122-marostegui.json |
[production] |
| 05:30 |
<ayounsi@cumin1002> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'email' for AS: 28306 |
[production] |
| 05:29 |
<ayounsi@cumin1002> |
START - Cookbook sre.network.peering with action 'email' for AS: 28306 |
[production] |
| 05:24 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2213.codfw.wmnet with reason: Long schema change |
[production] |
| 05:24 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2213.codfw.wmnet with reason: Long schema change |
[production] |
| 05:24 |
<marostegui> |
Deploy schema change on s5 codfw db2213 dbmaint T367856 |
[production] |
| 05:19 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depool db2213 T369478', diff saved to https://phabricator.wikimedia.org/P65923 and previous config saved to /var/cache/conftool/dbconfig/20240708-051935-root.json |
[production] |
| 05:18 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Promote db2123 to s5 primary T369478', diff saved to https://phabricator.wikimedia.org/P65922 and previous config saved to /var/cache/conftool/dbconfig/20240708-051840-root.json |
[production] |
| 05:18 |
<marostegui> |
Starting s5 codfw failover from db2213 to db2123 - T369478 |
[production] |
| 05:16 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1200', diff saved to https://phabricator.wikimedia.org/P65921 and previous config saved to /var/cache/conftool/dbconfig/20240708-051615-marostegui.json |
[production] |
| 05:16 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Remove db2123 from dump/slow', diff saved to https://phabricator.wikimedia.org/P65920 and previous config saved to /var/cache/conftool/dbconfig/20240708-051605-marostegui.json |
[production] |
| 05:03 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 25 hosts with reason: Primary switchover s5 T369478 |
[production] |
| 05:03 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Set db2123 with weight 0 T369478', diff saved to https://phabricator.wikimedia.org/P65919 and previous config saved to /var/cache/conftool/dbconfig/20240708-050301-root.json |
[production] |
| 05:02 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 25 hosts with reason: Primary switchover s5 T369478 |
[production] |
| 04:52 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1200', diff saved to https://phabricator.wikimedia.org/P65918 and previous config saved to /var/cache/conftool/dbconfig/20240708-045246-marostegui.json |
[production] |
| 04:37 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1200 (T367856)', diff saved to https://phabricator.wikimedia.org/P65917 and previous config saved to /var/cache/conftool/dbconfig/20240708-043738-marostegui.json |
[production] |
| 01:40 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1200 (T367856)', diff saved to https://phabricator.wikimedia.org/P65916 and previous config saved to /var/cache/conftool/dbconfig/20240708-014044-marostegui.json |
[production] |
| 01:40 |
<marostegui@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1200.eqiad.wmnet with reason: Maintenance |
[production] |
| 01:40 |
<marostegui@cumin1002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1200.eqiad.wmnet with reason: Maintenance |
[production] |
| 01:40 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1185 (T367856)', diff saved to https://phabricator.wikimedia.org/P65915 and previous config saved to /var/cache/conftool/dbconfig/20240708-014022-marostegui.json |
[production] |
| 01:25 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1185', diff saved to https://phabricator.wikimedia.org/P65914 and previous config saved to /var/cache/conftool/dbconfig/20240708-012515-marostegui.json |
[production] |
| 01:10 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1185', diff saved to https://phabricator.wikimedia.org/P65913 and previous config saved to /var/cache/conftool/dbconfig/20240708-011008-marostegui.json |
[production] |
| 00:55 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1185 (T367856)', diff saved to https://phabricator.wikimedia.org/P65912 and previous config saved to /var/cache/conftool/dbconfig/20240708-005501-marostegui.json |
[production] |