|
2023-03-07
ยง
|
| 15:02 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventstreams-internal: sync |
[production] |
| 15:01 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/eventstreams: sync |
[production] |
| 15:01 |
<akosiaris@cumin1001> |
START - Cookbook sre.hosts.downtime for 2:00:00 on kubernetes1021.eqiad.wmnet with reason: host reimage |
[production] |
| 15:01 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventstreams: sync |
[production] |
| 15:01 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/eventgate-main: sync |
[production] |
| 15:01 |
<sukhe> |
repooling dns1001: authdns-update can now be run again |
[production] |
| 15:01 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventgate-main: sync |
[production] |
| 15:01 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/eventgate-logging-external: sync |
[production] |
| 15:00 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventgate-logging-external: sync |
[production] |
| 15:00 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/eventgate-analytics-external: sync |
[production] |
| 15:00 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventgate-analytics-external: sync |
[production] |
| 15:00 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/eventgate-analytics: sync |
[production] |
| 15:00 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventgate-analytics: sync |
[production] |
| 15:00 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/echostore: sync |
[production] |
| 14:59 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/echostore: sync |
[production] |
| 14:59 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/developer-portal: sync |
[production] |
| 14:59 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/developer-portal: sync |
[production] |
| 14:59 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/datahub: sync on main |
[production] |
| 14:59 |
<hnowlan@puppetmaster1001> |
conftool action : set/pooled=yes; selector: name=restbase101[69].eqiad.wmnet |
[production] |
| 14:58 |
<hnowlan@puppetmaster1001> |
conftool action : set/pooled=yes; selector: name=restbase102[18].eqiad.wmnet |
[production] |
| 14:58 |
<hnowlan@puppetmaster1001> |
conftool action : set/pooled=yes; selector: name=restbase1031.eqiad.wmnet |
[production] |
| 14:58 |
<cmjohnson@cumin1001> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['cloudcephosd1038'] |
[production] |
| 14:58 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/datahub: sync on main |
[production] |
| 14:58 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/cxserver: sync |
[production] |
| 14:58 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/cxserver: sync |
[production] |
| 14:58 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/citoid: sync |
[production] |
| 14:57 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/citoid: sync |
[production] |
| 14:57 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: sync |
[production] |
| 14:57 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: sync |
[production] |
| 14:57 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/changeprop: sync |
[production] |
| 14:57 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/changeprop: sync |
[production] |
| 14:56 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/blubberoid: sync |
[production] |
| 14:56 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/blubberoid: sync |
[production] |
| 14:56 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/api-gateway: sync |
[production] |
| 14:56 |
<inflatador> |
bking@cumin2002 unban production row A elastic nodes from all clusters T329073 |
[production] |
| 14:56 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/api-gateway: sync |
[production] |
| 14:56 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/apertium: sync |
[production] |
| 14:55 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/apertium: sync |
[production] |
| 14:54 |
<akosiaris> |
T331126 toolhub deployed, https://toolhub.wikimedia.org/ operational again |
[production] |
| 14:53 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/toolhub: sync |
[production] |
| 14:53 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/services/toolhub: sync |
[production] |
| 14:52 |
<inflatador> |
bking@cumin2002 unban row A cloudelastic nodes T329073 |
[production] |
| 14:47 |
<akosiaris@cumin1001> |
START - Cookbook sre.hosts.reimage for host kubernetes1021.eqiad.wmnet with OS bullseye |
[production] |
| 14:45 |
<akosiaris> |
uncordon kubernetes{1005,1007,1008,1017,1018}.eqiad.wmnet T331126 |
[production] |
| 14:45 |
<akosiaris> |
uncordon kubernetes{1005,1007,1008,1017,1018}.eqiad.wmnet |
[production] |
| 14:44 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 14:43 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 14:43 |
<cmooney@cumin1001> |
END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for 238 hosts |
[production] |
| 14:43 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 14:43 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'apply'. |
[production] |