|
2022-08-02
ยง
|
| 15:06 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.downtime for 4:00:00 on mc-gp2002.codfw.wmnet with reason: Power down for PDU maintenance, T310070 |
[production] |
| 15:04 |
<bking@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on elastic2037.codfw.wmnet with reason: T309957 |
[production] |
| 15:04 |
<bking@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on elastic2037.codfw.wmnet with reason: T309957 |
[production] |
| 15:01 |
<sukhe@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 6 hosts with reason: shutdown for PDU upgrade |
[production] |
| 15:00 |
<sukhe@cumin2002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 6 hosts with reason: shutdown for PDU upgrade |
[production] |
| 14:59 |
<bking@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on elastic2025.codfw.wmnet with reason: T309957 |
[production] |
| 14:59 |
<bking@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on elastic2025.codfw.wmnet with reason: T309957 |
[production] |
| 14:58 |
<oblivian@puppetmaster1001> |
conftool action : set/pooled=false; selector: dnsdisc=(appservers|api)-ro,name=codfw |
[production] |
| 14:57 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3316', diff saved to https://phabricator.wikimedia.org/P32166 and previous config saved to /var/cache/conftool/dbconfig/20220802-145728-marostegui.json |
[production] |
| 14:54 |
<ryankemper@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host elastic2060.codfw.wmnet with OS bullseye |
[production] |
| 14:53 |
<jbond@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on puppetmaster1004.eqiad.wmnet with reason: host reimage |
[production] |
| 14:50 |
<moritzm> |
uploaded gnupg2 2.1.18-8~deb9u4+wmf1 to stretch-wikimedia |
[production] |
| 14:50 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on puppetmaster1004.eqiad.wmnet with reason: host reimage |
[production] |
| 14:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3316 (T312972)', diff saved to https://phabricator.wikimedia.org/P32164 and previous config saved to /var/cache/conftool/dbconfig/20220802-144222-marostegui.json |
[production] |
| 14:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1113:3316 (T312972)', diff saved to https://phabricator.wikimedia.org/P32163 and previous config saved to /var/cache/conftool/dbconfig/20220802-144013-marostegui.json |
[production] |
| 14:40 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:39 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:39 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3316 (T312972)', diff saved to https://phabricator.wikimedia.org/P32162 and previous config saved to /var/cache/conftool/dbconfig/20220802-143952-marostegui.json |
[production] |
| 14:37 |
<jbond@cumin1001> |
START - Cookbook sre.hosts.reimage for host puppetmaster1004.eqiad.wmnet with OS buster |
[production] |
| 14:32 |
<ryankemper@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on elastic2060.codfw.wmnet with reason: host reimage |
[production] |
| 14:28 |
<ryankemper@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on elastic2060.codfw.wmnet with reason: host reimage |
[production] |
| 14:24 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3316', diff saved to https://phabricator.wikimedia.org/P32161 and previous config saved to /var/cache/conftool/dbconfig/20220802-142446-marostegui.json |
[production] |
| 14:23 |
<Emperor> |
shutdown ms-be20[30,45,52] for PDU work T309957 |
[production] |
| 14:22 |
<mvernon@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on ms-be[2030,2045,2052].codfw.wmnet with reason: shutdown for PDU replacement |
[production] |
| 14:21 |
<mvernon@cumin2002> |
START - Cookbook sre.hosts.downtime for 1:00:00 on ms-be[2030,2045,2052].codfw.wmnet with reason: shutdown for PDU replacement |
[production] |
| 14:12 |
<ryankemper@cumin1001> |
START - Cookbook sre.hosts.reimage for host elastic2060.codfw.wmnet with OS bullseye |
[production] |
| 14:09 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3316', diff saved to https://phabricator.wikimedia.org/P32160 and previous config saved to /var/cache/conftool/dbconfig/20220802-140940-marostegui.json |
[production] |
| 14:05 |
<jbond@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host puppetmaster2004.codfw.wmnet with OS buster |
[production] |
| 14:04 |
<godog> |
grow sda/sdb 3 by 100G on thanos-be1001 - T314275 |
[production] |
| 14:03 |
<root@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on centrallog2002.codfw.wmnet with reason: pdu |
[production] |
| 14:03 |
<root@cumin1001> |
START - Cookbook sre.hosts.downtime for 4:00:00 on centrallog2002.codfw.wmnet with reason: pdu |
[production] |
| 14:01 |
<root@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 4:00:00 on prometheus2005.codfw.wmnet with reason: pdu |
[production] |
| 14:01 |
<root@cumin1001> |
START - Cookbook sre.hosts.downtime for 4:00:00 on prometheus2005.codfw.wmnet with reason: pdu |
[production] |
| 13:57 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp2030.codfw.wmnet,service=ats-tls |
[production] |
| 13:57 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp2032.codfw.wmnet,service=ats-be |
[production] |
| 13:57 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp2031.codfw.wmnet,service=ats-be |
[production] |
| 13:56 |
<godog> |
schedule poweroff for centrallog2002 at 16 utc - T310070 |
[production] |
| 13:54 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp203[3-4].codfw.wmnet,service=ats-be |
[production] |
| 13:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1098:3316 (T312972)', diff saved to https://phabricator.wikimedia.org/P32159 and previous config saved to /var/cache/conftool/dbconfig/20220802-135435-marostegui.json |
[production] |
| 13:53 |
<godog> |
depool and poweroff prometheus2005 - T310070 |
[production] |
| 13:53 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp203[3-4].codfw.wmnet,service=ats-tls |
[production] |
| 13:53 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp203[3-4].codfw.wmnet,service=ats-tls |
[production] |
| 13:53 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp203[3-4].codfw.wmnet,service=varnish-fe |
[production] |
| 13:52 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1098:3316 (T312972)', diff saved to https://phabricator.wikimedia.org/P32158 and previous config saved to /var/cache/conftool/dbconfig/20220802-135226-marostegui.json |
[production] |
| 13:52 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:52 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp203[1-2].codfw.wmnet,service=ats-tls |
[production] |
| 13:51 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:51 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3316 (T312972)', diff saved to https://phabricator.wikimedia.org/P32157 and previous config saved to /var/cache/conftool/dbconfig/20220802-135155-marostegui.json |
[production] |
| 13:51 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp203[1-2].codfw.wmnet,service=ats-tls |
[production] |
| 13:51 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp203[1-2].codfw.wmnet,service=varnish-fe |
[production] |