|
2023-02-27
ยง
|
| 12:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'es1022 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P44775 and previous config saved to /var/cache/conftool/dbconfig/20230227-124050-root.json |
[production] |
| 12:38 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool es1022 es2022 T330653', diff saved to https://phabricator.wikimedia.org/P44774 and previous config saved to /var/cache/conftool/dbconfig/20230227-123814-root.json |
[production] |
| 12:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1137 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P44773 and previous config saved to /var/cache/conftool/dbconfig/20230227-123742-root.json |
[production] |
| 12:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1137 (re)pooling @ 5%: Repooling', diff saved to https://phabricator.wikimedia.org/P44772 and previous config saved to /var/cache/conftool/dbconfig/20230227-123701-root.json |
[production] |
| 12:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1182 (re)pooling @ 3%: Repooling', diff saved to https://phabricator.wikimedia.org/P44771 and previous config saved to /var/cache/conftool/dbconfig/20230227-123636-root.json |
[production] |
| 12:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1137 T330653', diff saved to https://phabricator.wikimedia.org/P44770 and previous config saved to /var/cache/conftool/dbconfig/20230227-123514-root.json |
[production] |
| 12:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1143 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P44769 and previous config saved to /var/cache/conftool/dbconfig/20230227-123459-root.json |
[production] |
| 12:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1111 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P44768 and previous config saved to /var/cache/conftool/dbconfig/20230227-123454-root.json |
[production] |
| 12:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1200 (re)pooling @ 5%: Repooling', diff saved to https://phabricator.wikimedia.org/P44767 and previous config saved to /var/cache/conftool/dbconfig/20230227-123447-root.json |
[production] |
| 12:34 |
<cmooney@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host dse-k8s-worker1006.eqiad.wmnet with OS bullseye |
[production] |
| 12:31 |
<dcaro@cumin1001> |
START - Cookbook sre.hosts.reimage for host cloudcephosd1003.eqiad.wmnet with OS bullseye |
[production] |
| 12:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1200 db1111 db1168 db1143 T330653', diff saved to https://phabricator.wikimedia.org/P44766 and previous config saved to /var/cache/conftool/dbconfig/20230227-122804-root.json |
[production] |
| 12:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1182 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P44765 and previous config saved to /var/cache/conftool/dbconfig/20230227-122131-root.json |
[production] |
| 12:21 |
<cmooney@cumin1001> |
START - Cookbook sre.hosts.reimage for host dse-k8s-worker1006.eqiad.wmnet with OS bullseye |
[production] |
| 12:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1182', diff saved to https://phabricator.wikimedia.org/P44764 and previous config saved to /var/cache/conftool/dbconfig/20230227-121846-root.json |
[production] |
| 12:12 |
<moritzm> |
installing apr-util security updates on buster |
[production] |
| 12:10 |
<jbond@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on an-airflow1005.eqiad.wmnet with reason: host still been configuered - T327970 |
[production] |
| 12:10 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on an-airflow1005.eqiad.wmnet with reason: host still been configuered - T327970 |
[production] |
| 12:04 |
<cmooney@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host dse-k8s-worker1006.eqiad.wmnet with OS bullseye |
[production] |
| 12:00 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1127', diff saved to https://phabricator.wikimedia.org/P44763 and previous config saved to /var/cache/conftool/dbconfig/20230227-120002-root.json |
[production] |
| 11:59 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 5%: Repooling', diff saved to https://phabricator.wikimedia.org/P44762 and previous config saved to /var/cache/conftool/dbconfig/20230227-115947-root.json |
[production] |
| 11:53 |
<dcaro@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host cloudcephosd1003.eqiad.wmnet with OS bullseye |
[production] |
| 11:51 |
<cmooney@cumin1001> |
START - Cookbook sre.hosts.reimage for host dse-k8s-worker1006.eqiad.wmnet with OS bullseye |
[production] |
| 11:49 |
<vgutierrez> |
set "X-Content-Type-Options: nosniff" on upload.wm.o requests - T309787 |
[production] |
| 11:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 3%: Repooling', diff saved to https://phabricator.wikimedia.org/P44761 and previous config saved to /var/cache/conftool/dbconfig/20230227-114442-root.json |
[production] |
| 11:43 |
<hnowlan@deploy1002> |
Finished deploy [restbase/deploy@bcb0a69]: Add azwikimedia T317120 (duration: 01m 25s) |
[production] |
| 11:42 |
<hnowlan@deploy1002> |
Started deploy [restbase/deploy@bcb0a69]: Add azwikimedia T317120 |
[production] |
| 11:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depool db1127', diff saved to https://phabricator.wikimedia.org/P44760 and previous config saved to /var/cache/conftool/dbconfig/20230227-113130-root.json |
[production] |
| 11:29 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 1%: Repooling', diff saved to https://phabricator.wikimedia.org/P44759 and previous config saved to /var/cache/conftool/dbconfig/20230227-112937-root.json |
[production] |
| 11:29 |
<apergos> |
rsync public (huge!) xmldatadumps dir from dumpsdata1003 to dumpsdata1004; running from ariel screen session on dumpsdata1003, no bandwidth cap |
[production] |
| 11:28 |
<jbond@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on ms-fe2013.codfw.wmnet with reason: testing redfish T326848 |
[production] |
| 11:28 |
<jbond@cumin2002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on ms-fe2013.codfw.wmnet with reason: testing redfish T326848 |
[production] |
| 11:20 |
<elukey@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host dse-k8s-worker1006.eqiad.wmnet with OS bullseye |
[production] |
| 11:10 |
<apergos> |
rsync private xmldatadumps dir from dumpsdata1003 to dumpsdata1004; running from ariel screen session on dumpsdata1003, no bandwidth cap |
[production] |
| 11:08 |
<dcaro@cumin1001> |
START - Cookbook sre.hosts.reimage for host cloudcephosd1003.eqiad.wmnet with OS bullseye |
[production] |
| 11:07 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.reimage for host dse-k8s-worker1006.eqiad.wmnet with OS bullseye |
[production] |
| 11:05 |
<elukey@cumin1001> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host dse-k8s-worker1006.eqiad.wmnet with OS bullseye |
[production] |
| 11:04 |
<jbond@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cloudcephosd1003'] |
[production] |
| 10:59 |
<jbond@cumin1001> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=1) upgrade firmware for hosts ['cloudcephosd1003'] |
[production] |
| 10:54 |
<jbond@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cloudcephosd1003'] |
[production] |
| 10:48 |
<elukey@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 10:48 |
<elukey@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 10:43 |
<root@cumin1001> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['cloudcephosd1003'] |
[production] |
| 10:39 |
<root@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cloudcephosd1003'] |
[production] |
| 10:32 |
<marostegui> |
Restart eqiad sanitarium hosts T330502 |
[production] |
| 10:31 |
<root@cumin1001> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cloudcephosd1003'] |
[production] |
| 10:26 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.reimage for host dse-k8s-worker1006.eqiad.wmnet with OS bullseye |
[production] |
| 10:26 |
<marostegui> |
Restart codfw sanitarium hosts T330502 |
[production] |
| 10:23 |
<root@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cloudcephosd1003'] |
[production] |
| 10:23 |
<dcaro@cumin1001> |
END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host cloudcephosd1003.eqiad.wmnet with OS bullseye |
[production] |