|
2023-02-27
ยง
|
| 15:24 |
<cgoubert@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 15:21 |
<dcaro@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudcephosd1003.eqiad.wmnet with reason: host reimage |
[production] |
| 15:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P44884 and previous config saved to /var/cache/conftool/dbconfig/20230227-151836-root.json |
[production] |
| 15:18 |
<dcaro@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudcephosd1003.eqiad.wmnet with reason: host reimage |
[production] |
| 15:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1168 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P44883 and previous config saved to /var/cache/conftool/dbconfig/20230227-151826-root.json |
[production] |
| 15:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2180 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P44882 and previous config saved to /var/cache/conftool/dbconfig/20230227-151819-root.json |
[production] |
| 15:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2146 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P44881 and previous config saved to /var/cache/conftool/dbconfig/20230227-151813-root.json |
[production] |
| 15:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2178 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P44880 and previous config saved to /var/cache/conftool/dbconfig/20230227-151808-root.json |
[production] |
| 15:14 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2122 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P44878 and previous config saved to /var/cache/conftool/dbconfig/20230227-151434-root.json |
[production] |
| 15:13 |
<bking@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 15:13 |
<bking@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 15:12 |
<bking@deploy1002> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 15:12 |
<bking@deploy1002> |
helmfile [dse-k8s-eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 15:11 |
<inflatador> |
bking@deploy1002 applying https://gerrit.wikimedia.org/r/c/operations/deployment-charts/+/891577 on dse-k8s-cluster via helmfile |
[production] |
| 15:11 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on dse-k8s-worker1007.eqiad.wmnet with reason: host reimage |
[production] |
| 15:08 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on dse-k8s-worker1007.eqiad.wmnet with reason: host reimage |
[production] |
| 15:06 |
<dcaro@cumin1001> |
START - Cookbook sre.hosts.reimage for host cloudcephosd1003.eqiad.wmnet with OS bullseye |
[production] |
| 15:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2175 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P44877 and previous config saved to /var/cache/conftool/dbconfig/20230227-150535-root.json |
[production] |
| 15:04 |
<robh@cumin1001> |
END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host dumpsdata1006.eqiad.wmnet with OS bullseye |
[production] |
| 15:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P44876 and previous config saved to /var/cache/conftool/dbconfig/20230227-150331-root.json |
[production] |
| 15:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1168 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P44875 and previous config saved to /var/cache/conftool/dbconfig/20230227-150322-root.json |
[production] |
| 15:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2180 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P44874 and previous config saved to /var/cache/conftool/dbconfig/20230227-150315-root.json |
[production] |
| 15:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2146 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P44873 and previous config saved to /var/cache/conftool/dbconfig/20230227-150309-root.json |
[production] |
| 15:03 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2178 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P44872 and previous config saved to /var/cache/conftool/dbconfig/20230227-150304-root.json |
[production] |
| 15:01 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on ml-etcd2001.codfw.wmnet with reason: host reimage |
[production] |
| 14:59 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2122 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P44871 and previous config saved to /var/cache/conftool/dbconfig/20230227-145929-root.json |
[production] |
| 14:56 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on ml-etcd2001.codfw.wmnet with reason: host reimage |
[production] |
| 14:54 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.reimage for host dse-k8s-worker1007.eqiad.wmnet with OS bullseye |
[production] |
| 14:52 |
<robh@cumin1001> |
START - Cookbook sre.hosts.reimage for host dumpsdata1006.eqiad.wmnet with OS bullseye |
[production] |
| 14:52 |
<robh@cumin1001> |
END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host dumpsdata1006.eqiad.wmnet with OS bullseye |
[production] |
| 14:50 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2175 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P44870 and previous config saved to /var/cache/conftool/dbconfig/20230227-145030-root.json |
[production] |
| 14:49 |
<cmooney@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host dse-k8s-worker1006.eqiad.wmnet with OS bullseye |
[production] |
| 14:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P44869 and previous config saved to /var/cache/conftool/dbconfig/20230227-144826-root.json |
[production] |
| 14:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1168 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P44868 and previous config saved to /var/cache/conftool/dbconfig/20230227-144816-root.json |
[production] |
| 14:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2180 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P44867 and previous config saved to /var/cache/conftool/dbconfig/20230227-144810-root.json |
[production] |
| 14:48 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2146 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P44866 and previous config saved to /var/cache/conftool/dbconfig/20230227-144804-root.json |
[production] |
| 14:47 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2178 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P44865 and previous config saved to /var/cache/conftool/dbconfig/20230227-144759-root.json |
[production] |
| 14:45 |
<elukey@cumin1001> |
START - Cookbook sre.ganeti.reimage for host ml-etcd2001.codfw.wmnet with OS bullseye |
[production] |
| 14:44 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2122 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P44864 and previous config saved to /var/cache/conftool/dbconfig/20230227-144424-root.json |
[production] |
| 14:35 |
<claime> |
done live testing sre.switchdc.mediawiki.03-set-db-readonly and sre.switchdc.mediawiki.06-set-db-readwrite back to back - T330302 |
[production] |
| 14:35 |
<robh@cumin1001> |
START - Cookbook sre.hosts.reimage for host dumpsdata1006.eqiad.wmnet with OS bullseye |
[production] |
| 14:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2175 (re)pooling @ 50%: Repooling', diff saved to https://phabricator.wikimedia.org/P44863 and previous config saved to /var/cache/conftool/dbconfig/20230227-143525-root.json |
[production] |
| 14:35 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.06-set-db-readwrite (exit_code=0) |
[production] |
| 14:35 |
<cgoubert@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.06-set-db-readwrite |
[production] |
| 14:34 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.switchdc.mediawiki.03-set-db-readonly (exit_code=0) |
[production] |
| 14:34 |
<cgoubert@cumin1001> |
START - Cookbook sre.switchdc.mediawiki.03-set-db-readonly |
[production] |
| 14:34 |
<claime> |
live testing sre.switchdc.mediawiki.03-set-db-readonly and sre.switchdc.mediawiki.06-set-db-readwrite back to back - T330302 |
[production] |
| 14:33 |
<bking@cumin1001> |
START - Cookbook sre.wdqs.data-transfer |
[production] |
| 14:33 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1127 (re)pooling @ 25%: Repooling', diff saved to https://phabricator.wikimedia.org/P44862 and previous config saved to /var/cache/conftool/dbconfig/20230227-143321-root.json |
[production] |
| 14:33 |
<jbond@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 7 days, 0:00:00 on idm2001.wikimedia.org with reason: host still been configuered - T320797 |
[production] |