|
2023-03-07
ยง
|
| 15:23 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/sessionstore: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/sessionstore: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/recommendation-api: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/similar-users: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/recommendation-api: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/rdf-streaming-updater: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/similar-users: sync |
[production] |
| 15:21 |
<akosiaris@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host kubernetes1021.eqiad.wmnet with OS bullseye |
[production] |
| 15:21 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/tegola-vector-tiles: sync |
[production] |
| 15:21 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/rdf-streaming-updater: sync |
[production] |
| 15:21 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/push-notifications: sync |
[production] |
| 15:21 |
<cmjohnson@cumin1001> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['cloudcephosd1039'] |
[production] |
| 15:21 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/tegola-vector-tiles: sync |
[production] |
| 15:20 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cloudcephosd1039'] |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/termbox: sync |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/push-notifications: sync |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/proton: sync |
[production] |
| 15:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1105:3312 (T329260)', diff saved to https://phabricator.wikimedia.org/P45198 and previous config saved to /var/cache/conftool/dbconfig/20230307-152037-marostegui.json |
[production] |
| 15:20 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cloudcephosd1039'] |
[production] |
| 15:20 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/termbox: sync |
[production] |
| 15:20 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/proton: sync |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: sync |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/wikifeeds: sync |
[production] |
| 15:19 |
<Emperor> |
pool thanos-fe1001 T329073 |
[production] |
| 15:19 |
<sukhe@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on durum1001.eqiad.wmnet with reason: host reimage |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/wikifeeds: sync |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/zotero: sync |
[production] |
| 15:19 |
<mvernon@cumin1001> |
conftool action : set/pooled=no; selector: name=thanos-fe1002.eqiad.wmnet,service=thanos-web |
[production] |
| 15:18 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/zotero: sync |
[production] |
| 15:18 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2097.codfw.wmnet with reason: Maintenance |
[production] |
| 15:18 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2097.codfw.wmnet with reason: Maintenance |
[production] |
| 15:17 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2099.codfw.wmnet with reason: Maintenance |
[production] |
| 15:17 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2099.codfw.wmnet with reason: Maintenance |
[production] |
| 15:16 |
<sukhe@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on durum1001.eqiad.wmnet with reason: host reimage |
[production] |
| 15:16 |
<Emperor> |
pool ms-fe1009 T329073 |
[production] |
| 15:16 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:16 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1102.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:16 |
<Emperor> |
pool moss-fe1001 T329073 |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-web: sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] [main] DONE helmfile.d/services/mw-jobrunner : sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] [canary] DONE helmfile.d/services/mw-jobrunner : sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] [canary] START helmfile.d/services/mw-jobrunner : sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] [main] START helmfile.d/services/mw-jobrunner : sync |
[production] |
| 15:15 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-debug: sync |
[production] |
| 15:11 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mw-debug: sync |
[production] |
| 15:11 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-api-int: sync |
[production] |