|
2022-03-18
ยง
|
| 10:54 |
<btullis@cumin1001> |
START - Cookbook sre.kafka.roll-restart-brokers for Kafka A:kafka-test-eqiad cluster: Roll restart of jvm daemons for openjdk upgrade. |
[production] |
| 10:52 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host dumpsdata1007.eqiad.wmnet |
[production] |
| 10:52 |
<akosiaris> |
drain kubernetes200[1-4] T303045 |
[production] |
| 10:51 |
<akosiaris> |
depool kubernetes200[1-4] T303045 |
[production] |
| 10:50 |
<akosiaris@cumin1001> |
conftool action : set/pooled=no; selector: name=kubernetes2004.codfw.wmnet |
[production] |
| 10:50 |
<akosiaris@cumin1001> |
conftool action : set/pooled=no; selector: name=kubernetes2003.codfw.wmnet |
[production] |
| 10:50 |
<akosiaris@cumin1001> |
conftool action : set/pooled=no; selector: name=kubernetes2002.codfw.wmnet |
[production] |
| 10:50 |
<akosiaris@cumin1001> |
conftool action : set/pooled=no; selector: name=kubernetes2001.codfw.wmnet |
[production] |
| 10:01 |
<akosiaris> |
drain kubernetes100[1-4] T303044 |
[production] |
| 09:54 |
<akosiaris> |
depool kubernetes100[1-4] from pybal T303044 |
[production] |
| 09:52 |
<akosiaris@cumin1001> |
conftool action : set/pooled=no; selector: name=kubernetes1004.eqiad.wmnet |
[production] |
| 09:52 |
<akosiaris@cumin1001> |
conftool action : set/pooled=no; selector: name=kubernetes1003.eqiad.wmnet |
[production] |
| 09:52 |
<akosiaris@cumin1001> |
conftool action : set/pooled=no; selector: name=kubernetes1002.eqiad.wmnet |
[production] |
| 09:52 |
<akosiaris@cumin1001> |
conftool action : set/pooled=no; selector: name=kubernetes1001.eqiad.wmnet |
[production] |
| 09:42 |
<akosiaris> |
uncordon kubernetes1018-1022. T293728. Nodes are live, ready to receive workloads and traffic. |
[production] |
| 09:37 |
<akosiaris> |
pool kubernetes1018-1022 in pybal. T293728 |
[production] |
| 09:37 |
<akosiaris> |
pool kubernetes1018-1022 in pybal. |
[production] |
| 09:37 |
<akosiaris@cumin1001> |
conftool action : set/pooled=yes; selector: name=kubernetes1022.eqiad.wmnet |
[production] |
| 09:37 |
<akosiaris@cumin1001> |
conftool action : set/pooled=yes; selector: name=kubernetes1021.eqiad.wmnet |
[production] |
| 09:37 |
<akosiaris@cumin1001> |
conftool action : set/pooled=yes; selector: name=kubernetes1020.eqiad.wmnet |
[production] |
| 09:37 |
<akosiaris@cumin1001> |
conftool action : set/pooled=yes; selector: name=kubernetes1019.eqiad.wmnet |
[production] |
| 09:37 |
<akosiaris@cumin1001> |
conftool action : set/pooled=yes; selector: name=kubernetes1018.eqiad.wmnet |
[production] |
| 09:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1160 (T300775)', diff saved to https://phabricator.wikimedia.org/P22827 and previous config saved to /var/cache/conftool/dbconfig/20220318-093543-marostegui.json |
[production] |
| 09:35 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 3 days, 0:00:00 on db1160.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:35 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 3 days, 0:00:00 on db1160.eqiad.wmnet with reason: Maintenance |
[production] |
| 09:35 |
<akosiaris@cumin1001> |
conftool action : set/weight=10; selector: name=kubernetes1022.eqiad.wmnet |
[production] |
| 09:35 |
<akosiaris@cumin1001> |
conftool action : set/weight=10; selector: name=kubernetes1021.eqiad.wmnet |
[production] |
| 09:35 |
<akosiaris@cumin1001> |
conftool action : set/weight=10; selector: name=kubernetes1020.eqiad.wmnet |
[production] |
| 09:35 |
<akosiaris@cumin1001> |
conftool action : set/weight=10; selector: name=kubernetes1019.eqiad.wmnet |
[production] |
| 09:35 |
<akosiaris@cumin1001> |
conftool action : set/weight=10; selector: name=kubernetes1018.eqiad.wmnet |
[production] |
| 09:10 |
<kharlan@deploy1002> |
helmfile [staging] DONE helmfile.d/services/linkrecommendation: apply |
[production] |
| 09:08 |
<kharlan@deploy1002> |
helmfile [staging] START helmfile.d/services/linkrecommendation: apply |
[production] |
| 08:55 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1169 (T298557)', diff saved to https://phabricator.wikimedia.org/P22826 and previous config saved to /var/cache/conftool/dbconfig/20220318-085517-marostegui.json |
[production] |
| 08:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1169', diff saved to https://phabricator.wikimedia.org/P22825 and previous config saved to /var/cache/conftool/dbconfig/20220318-084012-marostegui.json |
[production] |
| 08:25 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1169', diff saved to https://phabricator.wikimedia.org/P22824 and previous config saved to /var/cache/conftool/dbconfig/20220318-082507-marostegui.json |
[production] |
| 08:10 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1169 (T298557)', diff saved to https://phabricator.wikimedia.org/P22823 and previous config saved to /var/cache/conftool/dbconfig/20220318-081002-marostegui.json |
[production] |
| 07:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1179 (re)pooling @ 100%: After upgrade', diff saved to https://phabricator.wikimedia.org/P22822 and previous config saved to /var/cache/conftool/dbconfig/20220318-072852-root.json |
[production] |
| 07:18 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1169 (T298557)', diff saved to https://phabricator.wikimedia.org/P22821 and previous config saved to /var/cache/conftool/dbconfig/20220318-071758-marostegui.json |
[production] |
| 07:17 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1169.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:17 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1169.eqiad.wmnet with reason: Maintenance |
[production] |
| 07:17 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1164 (T298557)', diff saved to https://phabricator.wikimedia.org/P22820 and previous config saved to /var/cache/conftool/dbconfig/20220318-071750-marostegui.json |
[production] |
| 07:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1179 (re)pooling @ 75%: After upgrade', diff saved to https://phabricator.wikimedia.org/P22819 and previous config saved to /var/cache/conftool/dbconfig/20220318-071348-root.json |
[production] |
| 07:02 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1164', diff saved to https://phabricator.wikimedia.org/P22818 and previous config saved to /var/cache/conftool/dbconfig/20220318-070245-marostegui.json |
[production] |
| 06:58 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1179 (re)pooling @ 50%: After upgrade', diff saved to https://phabricator.wikimedia.org/P22817 and previous config saved to /var/cache/conftool/dbconfig/20220318-065844-root.json |
[production] |
| 06:47 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1164', diff saved to https://phabricator.wikimedia.org/P22816 and previous config saved to /var/cache/conftool/dbconfig/20220318-064740-marostegui.json |
[production] |
| 06:43 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1179 (re)pooling @ 25%: After upgrade', diff saved to https://phabricator.wikimedia.org/P22815 and previous config saved to /var/cache/conftool/dbconfig/20220318-064340-root.json |
[production] |
| 06:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1146:3312 (re)pooling @ 100%: After schema change ', diff saved to https://phabricator.wikimedia.org/P22814 and previous config saved to /var/cache/conftool/dbconfig/20220318-063631-root.json |
[production] |
| 06:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1096:3316 (re)pooling @ 100%: After schema change ', diff saved to https://phabricator.wikimedia.org/P22813 and previous config saved to /var/cache/conftool/dbconfig/20220318-063524-root.json |
[production] |
| 06:32 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1164 (T298557)', diff saved to https://phabricator.wikimedia.org/P22812 and previous config saved to /var/cache/conftool/dbconfig/20220318-063235-marostegui.json |
[production] |
| 06:28 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1179 (re)pooling @ 10%: After upgrade', diff saved to https://phabricator.wikimedia.org/P22811 and previous config saved to /var/cache/conftool/dbconfig/20220318-062836-root.json |
[production] |