|
2023-08-29
ยง
|
| 19:51 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1160 (T343718)', diff saved to https://phabricator.wikimedia.org/P51950 and previous config saved to /var/cache/conftool/dbconfig/20230829-195154-ladsgroup.json |
[production] |
| 19:36 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1160', diff saved to https://phabricator.wikimedia.org/P51949 and previous config saved to /var/cache/conftool/dbconfig/20230829-193648-ladsgroup.json |
[production] |
| 19:35 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host restbase1033.eqiad.wmnet |
[production] |
| 19:32 |
<ayounsi@cumin1001> |
END (ERROR) - Cookbook sre.network.tls (exit_code=97) for network device asw2-c2-eqiad |
[production] |
| 19:32 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device asw2-c2-eqiad |
[production] |
| 19:32 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device ssw1-f1-eqiad |
[production] |
| 19:30 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device ssw1-f1-eqiad |
[production] |
| 19:30 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device ssw1-e1-eqiad |
[production] |
| 19:27 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device ssw1-e1-eqiad |
[production] |
| 19:27 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-f3-eqiad |
[production] |
| 19:26 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1033.eqiad.wmnet |
[production] |
| 19:25 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device lsw1-f3-eqiad |
[production] |
| 19:25 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-f2-eqiad |
[production] |
| 19:24 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cloudservices1006.eqiad.wmnet with OS bullseye |
[production] |
| 19:24 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002" |
[production] |
| 19:23 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device lsw1-f2-eqiad |
[production] |
| 19:23 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-f1-eqiad |
[production] |
| 19:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1160', diff saved to https://phabricator.wikimedia.org/P51948 and previous config saved to /var/cache/conftool/dbconfig/20230829-192141-ladsgroup.json |
[production] |
| 19:20 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device lsw1-f1-eqiad |
[production] |
| 19:20 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-e3-eqiad |
[production] |
| 19:18 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device lsw1-e3-eqiad |
[production] |
| 19:18 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-e2-eqiad |
[production] |
| 19:18 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host restbase1026.eqiad.wmnet |
[production] |
| 19:18 |
<pt1979@cumin2002> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002" |
[production] |
| 19:16 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device lsw1-e2-eqiad |
[production] |
| 19:16 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.tls (exit_code=0) for network device lsw1-e1-eqiad |
[production] |
| 19:13 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.tls for network device lsw1-e1-eqiad |
[production] |
| 19:11 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.debug (exit_code=0) for Netbox circuit ID 173 |
[production] |
| 19:11 |
<eileen> |
civicrm upgraded from d13e6e0c to fc5c73db |
[production] |
| 19:10 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.debug for Netbox circuit ID 173 |
[production] |
| 19:10 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1026.eqiad.wmnet |
[production] |
| 19:09 |
<eileen> |
civicrm upgraded from d13e6e0c to fc5c73db |
[production] |
| 19:07 |
<zabe@deploy1002> |
Finished scap: update interwiki cache (duration: 07m 08s) |
[production] |
| 19:06 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1160 (T343718)', diff saved to https://phabricator.wikimedia.org/P51947 and previous config saved to /var/cache/conftool/dbconfig/20230829-190635-ladsgroup.json |
[production] |
| 19:01 |
<pt1979@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudservices1006.eqiad.wmnet with reason: host reimage |
[production] |
| 19:01 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host restbase1025.eqiad.wmnet |
[production] |
| 19:00 |
<zabe@deploy1002> |
Started scap: update interwiki cache |
[production] |
| 18:56 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudservices1006.eqiad.wmnet with reason: host reimage |
[production] |
| 18:55 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.reimage for host cloudservices1006.eqiad.wmnet with OS bullseye |
[production] |
| 18:55 |
<pt1979@cumin2002> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host cloudservices1006.eqiad.wmnet with OS bullseye |
[production] |
| 18:54 |
<pt1979@cumin2002> |
START - Cookbook sre.hosts.reimage for host cloudservices1006.eqiad.wmnet with OS bullseye |
[production] |
| 18:52 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1025.eqiad.wmnet |
[production] |
| 18:37 |
<aokoth@cumin1001> |
END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host vrts1002.eqiad.wmnet |
[production] |
| 18:37 |
<aokoth@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host vrts1002.eqiad.wmnet with OS bullseye |
[production] |
| 18:32 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host restbase1032.eqiad.wmnet |
[production] |
| 18:24 |
<eevans@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host restbase1032.eqiad.wmnet |
[production] |
| 18:24 |
<aokoth@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on vrts1002.eqiad.wmnet with reason: host reimage |
[production] |
| 18:22 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2155 (T343718)', diff saved to https://phabricator.wikimedia.org/P51946 and previous config saved to /var/cache/conftool/dbconfig/20230829-182251-ladsgroup.json |
[production] |
| 18:22 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on db2187.codfw.wmnet with reason: Maintenance |
[production] |
| 18:22 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on db2187.codfw.wmnet with reason: Maintenance |
[production] |