|
2022-03-07
ยง
|
| 23:50 |
<jhathaway@cumin1001> |
START - Cookbook sre.hosts.downtime for 0:15:00 on mx2001.wikimedia.org with reason: reboot |
[production] |
| 23:49 |
<jhathaway@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on mx1001.wikimedia.org with reason: reboot |
[production] |
| 23:49 |
<jhathaway@cumin1001> |
START - Cookbook sre.hosts.downtime for 0:15:00 on mx1001.wikimedia.org with reason: reboot |
[production] |
| 23:40 |
<jhathaway@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:15:00 on mirror1001.wikimedia.org with reason: reboot |
[production] |
| 23:40 |
<jhathaway@cumin1001> |
START - Cookbook sre.hosts.downtime for 0:15:00 on mirror1001.wikimedia.org with reason: reboot |
[production] |
| 22:37 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cloudservices1003.wikimedia.org with OS bullseye |
[production] |
| 22:28 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudservices1003.wikimedia.org with reason: host reimage |
[production] |
| 22:26 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudservices1003.wikimedia.org with reason: host reimage |
[production] |
| 22:25 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.reimage for host cloudservices1003.wikimedia.org with OS bullseye |
[production] |
| 22:21 |
<andrew@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host cloudservices1003.wikimedia.org with OS bullseye |
[production] |
| 22:20 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudservices1003.wikimedia.org with reason: host reimage |
[production] |
| 22:18 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudservices1003.wikimedia.org with reason: host reimage |
[production] |
| 21:49 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.reimage for host cloudservices1003.wikimedia.org with OS bullseye |
[production] |
| 21:38 |
<urbanecm> |
UTC late B&C window done |
[production] |
| 21:37 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 21:37 |
<urbanecm@deploy1002> |
Synchronized php-1.38.0-wmf.24/skins/Vector/includes/SkinVector.php: eac551c: Fix language alert regression (T302018) (duration: 00m 50s) |
[production] |
| 21:36 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 21:36 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 21:35 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 21:22 |
<eileen> |
config aa7dcd88 -> 16fa8e1c |
[production] |
| 20:39 |
<andrew@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host cloudservices1003.wikimedia.org with OS bullseye |
[production] |
| 20:16 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudservices1003.wikimedia.org with reason: host reimage |
[production] |
| 20:13 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudservices1003.wikimedia.org with reason: host reimage |
[production] |
| 19:49 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.reimage for host cloudservices1003.wikimedia.org with OS bullseye |
[production] |
| 18:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3317 (T300381)', diff saved to https://phabricator.wikimedia.org/P22016 and previous config saved to /var/cache/conftool/dbconfig/20220307-181310-marostegui.json |
[production] |
| 17:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3317', diff saved to https://phabricator.wikimedia.org/P22015 and previous config saved to /var/cache/conftool/dbconfig/20220307-175805-marostegui.json |
[production] |
| 17:55 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubestage2002.codfw.wmnet |
[production] |
| 17:49 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubestage2002.codfw.wmnet |
[production] |
| 17:47 |
<jayme@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reboot-single (exit_code=99) for host kubestage2002.codfw.wmnet |
[production] |
| 17:47 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubestage2002.codfw.wmnet |
[production] |
| 17:44 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubestage2001.codfw.wmnet |
[production] |
| 17:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3317', diff saved to https://phabricator.wikimedia.org/P22014 and previous config saved to /var/cache/conftool/dbconfig/20220307-174300-marostegui.json |
[production] |
| 17:36 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubestage2001.codfw.wmnet |
[production] |
| 17:32 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cloudservices1004.wikimedia.org with OS bullseye |
[production] |
| 17:29 |
<jayme@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host kubernetes1022.eqiad.wmnet |
[production] |
| 17:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1170:3317 (T300381)', diff saved to https://phabricator.wikimedia.org/P22013 and previous config saved to /var/cache/conftool/dbconfig/20220307-172755-marostegui.json |
[production] |
| 17:24 |
<jayme@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host kubernetes1022.eqiad.wmnet |
[production] |
| 17:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1170:3317 (T300381)', diff saved to https://phabricator.wikimedia.org/P22012 and previous config saved to /var/cache/conftool/dbconfig/20220307-172134-marostegui.json |
[production] |
| 17:21 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:21 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1170.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317 (T300381)', diff saved to https://phabricator.wikimedia.org/P22011 and previous config saved to /var/cache/conftool/dbconfig/20220307-172126-marostegui.json |
[production] |
| 17:20 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on cp5004.eqsin.wmnet with reason: HW issues see T303043 |
[production] |
| 17:20 |
<vgutierrez@cumin1001> |
START - Cookbook sre.hosts.downtime for 30 days, 0:00:00 on cp5004.eqsin.wmnet with reason: HW issues see T303043 |
[production] |
| 17:09 |
<andrew@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudservices1004.wikimedia.org with reason: host reimage |
[production] |
| 17:07 |
<vgutierrez@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cp3058.esams.wmnet with OS buster |
[production] |
| 17:07 |
<vgutierrez> |
pool cp3058 with HAProxy as TLS termination layer - T290005 |
[production] |
| 17:06 |
<andrew@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudservices1004.wikimedia.org with reason: host reimage |
[production] |
| 17:06 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1101:3317', diff saved to https://phabricator.wikimedia.org/P22010 and previous config saved to /var/cache/conftool/dbconfig/20220307-170622-marostegui.json |
[production] |
| 17:03 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host netflow6001.drmrs.wmnet |
[production] |
| 16:59 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host netflow6001.drmrs.wmnet |
[production] |