|
2023-03-07
ยง
|
| 15:20 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cloudcephosd1039'] |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/termbox: sync |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/push-notifications: sync |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/proton: sync |
[production] |
| 15:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1105:3312 (T329260)', diff saved to https://phabricator.wikimedia.org/P45198 and previous config saved to /var/cache/conftool/dbconfig/20230307-152037-marostegui.json |
[production] |
| 15:20 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cloudcephosd1039'] |
[production] |
| 15:20 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/termbox: sync |
[production] |
| 15:20 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/proton: sync |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: sync |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/wikifeeds: sync |
[production] |
| 15:19 |
<Emperor> |
pool thanos-fe1001 T329073 |
[production] |
| 15:19 |
<sukhe@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on durum1001.eqiad.wmnet with reason: host reimage |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/wikifeeds: sync |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/zotero: sync |
[production] |
| 15:19 |
<mvernon@cumin1001> |
conftool action : set/pooled=no; selector: name=thanos-fe1002.eqiad.wmnet,service=thanos-web |
[production] |
| 15:18 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/zotero: sync |
[production] |
| 15:18 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2097.codfw.wmnet with reason: Maintenance |
[production] |
| 15:18 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2097.codfw.wmnet with reason: Maintenance |
[production] |
| 15:17 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2099.codfw.wmnet with reason: Maintenance |
[production] |
| 15:17 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2099.codfw.wmnet with reason: Maintenance |
[production] |
| 15:16 |
<sukhe@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on durum1001.eqiad.wmnet with reason: host reimage |
[production] |
| 15:16 |
<Emperor> |
pool ms-fe1009 T329073 |
[production] |
| 15:16 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:16 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:16 |
<Emperor> |
pool moss-fe1001 T329073 |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-web: sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] [main] DONE helmfile.d/services/mw-jobrunner : sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] [canary] DONE helmfile.d/services/mw-jobrunner : sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] [canary] START helmfile.d/services/mw-jobrunner : sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] [main] START helmfile.d/services/mw-jobrunner : sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-debug: sync |
[production] |
| 15:11 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-debug: sync |
[production] |
| 15:11 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-int: sync |
[production] |
| 15:11 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-api-int: sync |
[production] |
| 15:11 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cloudcephosd1039'] |
[production] |
| 15:11 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-ext: sync |
[production] |
| 15:09 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cloudcephosd1038'] |
[production] |
| 15:06 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-api-ext: sync |
[production] |
| 15:06 |
<sukhe@cumin2002> |
START - Cookbook sre.ganeti.reimage for host durum1001.eqiad.wmnet with OS bullseye |
[production] |
| 15:06 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mobileapps: sync |
[production] |
| 15:06 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mobileapps: sync |
[production] |
| 15:06 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/miscweb: sync |
[production] |
| 15:04 |
<akosiaris@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on kubernetes1021.eqiad.wmnet with reason: host reimage |
[production] |
| 15:04 |
<bblack> |
dns1001 - restarted prometheus-bird-exporter |
[production] |
| 15:04 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/miscweb: sync |
[production] |
| 15:04 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mathoid: sync |
[production] |
| 15:04 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mathoid: sync |
[production] |
| 15:04 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/linkrecommendation: sync |
[production] |