|
2023-03-07
ยง
|
| 15:26 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/toolhub: sync |
[production] |
| 15:26 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/toolhub: sync |
[production] |
| 15:26 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/thumbor: sync |
[production] |
| 15:25 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1105:3312 (T329260)', diff saved to https://phabricator.wikimedia.org/P45199 and previous config saved to /var/cache/conftool/dbconfig/20230307-152545-marostegui.json |
[production] |
| 15:25 |
<herron@cumin1001> |
START - Cookbook sre.kafka.roll-restart-brokers for Kafka A:kafka-logging-eqiad cluster: Roll restart of jvm daemons. |
[production] |
| 15:25 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/thumbor: sync |
[production] |
| 15:25 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/termbox: sync |
[production] |
| 15:25 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/termbox: sync |
[production] |
| 15:25 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/tegola-vector-tiles: sync |
[production] |
| 15:25 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/tegola-vector-tiles: sync |
[production] |
| 15:25 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/similar-users: sync |
[production] |
| 15:25 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/similar-users: sync |
[production] |
| 15:24 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-timeline: sync |
[production] |
| 15:24 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-timeline: sync |
[production] |
| 15:24 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-syntaxhighlight: sync |
[production] |
| 15:24 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-syntaxhighlight: sync |
[production] |
| 15:24 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-media: sync |
[production] |
| 15:24 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-media: sync |
[production] |
| 15:23 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox-constraints: sync |
[production] |
| 15:23 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox-constraints: sync |
[production] |
| 15:23 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/shellbox: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/shellbox: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/sessionstore: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/sessionstore: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/recommendation-api: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/similar-users: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/recommendation-api: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/rdf-streaming-updater: sync |
[production] |
| 15:22 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/similar-users: sync |
[production] |
| 15:21 |
<akosiaris@cumin1001> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host kubernetes1021.eqiad.wmnet with OS bullseye |
[production] |
| 15:21 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/tegola-vector-tiles: sync |
[production] |
| 15:21 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/rdf-streaming-updater: sync |
[production] |
| 15:21 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/push-notifications: sync |
[production] |
| 15:21 |
<cmjohnson@cumin1001> |
END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=99) upgrade firmware for hosts ['cloudcephosd1039'] |
[production] |
| 15:21 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/tegola-vector-tiles: sync |
[production] |
| 15:20 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cloudcephosd1039'] |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/termbox: sync |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/push-notifications: sync |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/proton: sync |
[production] |
| 15:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1105:3312 (T329260)', diff saved to https://phabricator.wikimedia.org/P45198 and previous config saved to /var/cache/conftool/dbconfig/20230307-152037-marostegui.json |
[production] |
| 15:20 |
<cmjohnson@cumin1001> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts ['cloudcephosd1039'] |
[production] |
| 15:20 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:20 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/termbox: sync |
[production] |
| 15:20 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1105.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/proton: sync |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mw-web: sync |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/wikifeeds: sync |
[production] |
| 15:19 |
<Emperor> |
pool thanos-fe1001 T329073 |
[production] |
| 15:19 |
<sukhe@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on durum1001.eqiad.wmnet with reason: host reimage |
[production] |
| 15:19 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/wikifeeds: sync |
[production] |