|
2023-06-15
ยง
|
| 13:08 |
<jgiannelos@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/wikifeeds: apply |
[production] |
| 13:08 |
<elukey@cumin1001> |
START - Cookbook sre.ores.roll-restart-workers for ORES codfw cluster: Roll restart of ORES's daemons. |
[production] |
| 13:07 |
<jgiannelos@deploy1002> |
helmfile [codfw] START helmfile.d/services/wikifeeds: apply |
[production] |
| 13:05 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.ores.roll-restart-workers (exit_code=0) for ORES eqiad cluster: Roll restart of ORES's daemons. |
[production] |
| 12:59 |
<jgiannelos@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/wikifeeds: apply |
[production] |
| 12:58 |
<jgiannelos@deploy1002> |
helmfile [eqiad] START helmfile.d/services/wikifeeds: apply |
[production] |
| 12:57 |
<jgiannelos@deploy1002> |
helmfile [staging] DONE helmfile.d/services/wikifeeds: apply |
[production] |
| 12:57 |
<jgiannelos@deploy1002> |
helmfile [staging] START helmfile.d/services/wikifeeds: apply |
[production] |
| 12:53 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp5022.eqsin.wmnet |
[production] |
| 12:51 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp5030.eqsin.wmnet |
[production] |
| 12:48 |
<stevemunene@cumin1001> |
START - Cookbook sre.hadoop.roll-restart-masters restart masters for Hadoop analytics cluster: Restart of jvm daemons. |
[production] |
| 12:45 |
<elukey@cumin1001> |
START - Cookbook sre.ores.roll-restart-workers for ORES eqiad cluster: Roll restart of ORES's daemons. |
[production] |
| 12:40 |
<fabfur> |
reboot cp5022 and cp5030 for kernel upgrade (T335835) |
[production] |
| 12:40 |
<fabfur@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp5022.eqsin.wmnet |
[production] |
| 12:40 |
<fabfur@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp5030.eqsin.wmnet |
[production] |
| 12:35 |
<moritzm> |
installing ffmpeg security updates |
[production] |
| 12:34 |
<joal@deploy1002> |
Finished deploy [airflow-dags/analytics@d458338]: (no justification provided) (duration: 00m 09s) |
[production] |
| 12:34 |
<joal@deploy1002> |
Started deploy [airflow-dags/analytics@d458338]: (no justification provided) |
[production] |
| 12:27 |
<moritzm> |
installing containerd security updates |
[production] |
| 12:18 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp5029.eqsin.wmnet |
[production] |
| 12:17 |
<fabfur@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host cp5021.eqsin.wmnet |
[production] |
| 12:14 |
<aborrero@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cloudservices2004-dev.codfw.wmnet with reason: host reimage |
[production] |
| 12:11 |
<aborrero@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cloudservices2004-dev.codfw.wmnet with reason: host reimage |
[production] |
| 12:07 |
<fabfur> |
reboot cp5021 and cp5029 for kernel upgrade (T335835) |
[production] |
| 12:06 |
<fabfur@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp5021.eqsin.wmnet |
[production] |
| 12:06 |
<fabfur@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host cp5029.eqsin.wmnet |
[production] |
| 12:02 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for parse1002.eqiad.wmnet |
[production] |
| 12:02 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.remove-downtime for parse1002.eqiad.wmnet |
[production] |
| 11:58 |
<moritzm> |
restarting exim on lists1001 |
[production] |
| 11:52 |
<aborrero@cumin2002> |
START - Cookbook sre.hosts.reimage for host cloudservices2004-dev.codfw.wmnet with OS bullseye |
[production] |
| 11:52 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.maps.roll-restart (exit_code=0) rolling restart_daemons on A:maps-replica-eqiad |
[production] |
| 11:51 |
<claime> |
Repooled parse1002.eqiad.wmnet after powercycle |
[production] |
| 11:49 |
<moritzm> |
restarting slapd on seagorgium/serpens |
[production] |
| 11:48 |
<jmm@cumin2002> |
START - Cookbook sre.maps.roll-restart rolling restart_daemons on A:maps-replica-eqiad |
[production] |
| 11:48 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.maps.roll-restart (exit_code=0) rolling restart_daemons on A:maps-replica-codfw |
[production] |
| 11:46 |
<ladsgroup@deploy1002> |
Finished scap: Backport for [[gerrit:930560|Switch five large wikis to extlinks read new (T335343)]] (duration: 09m 10s) |
[production] |
| 11:45 |
<jmm@cumin2002> |
START - Cookbook sre.maps.roll-restart rolling restart_daemons on A:maps-replica-codfw |
[production] |
| 11:40 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.aqs.roll-restart-reboot (exit_code=0) rolling restart_daemons on A:aqs-eqiad |
[production] |
| 11:40 |
<cgoubert@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on parse1002.eqiad.wmnet with reason: Powercycle |
[production] |
| 11:40 |
<cgoubert@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on parse1002.eqiad.wmnet with reason: Powercycle |
[production] |
| 11:39 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas (exit_code=0) rolling restart_daemons on A:schema |
[production] |
| 11:39 |
<claime> |
parse1002 not responding to ssh or console, depooled |
[production] |
| 11:38 |
<ladsgroup@deploy1002> |
ladsgroup: Backport for [[gerrit:930560|Switch five large wikis to extlinks read new (T335343)]] synced to the testservers: mwdebug2001.codfw.wmnet, mwdebug1002.eqiad.wmnet, mwdebug2002.codfw.wmnet, mwdebug1001.eqiad.wmnet |
[production] |
| 11:37 |
<cgoubert@cumin1001> |
conftool action : set/pooled=inactive; selector: dc=eqiad,name=parse1002.eqiad.wmnet |
[production] |
| 11:37 |
<jmm@cumin2002> |
START - Cookbook sre.misc-clusters.roll-restart-reboot-eventschemas rolling restart_daemons on A:schema |
[production] |
| 11:37 |
<ladsgroup@deploy1002> |
Started scap: Backport for [[gerrit:930560|Switch five large wikis to extlinks read new (T335343)]] |
[production] |
| 11:32 |
<ladsgroup@deploy1002> |
Finished scap: Backport for [[gerrit:930192|Remove nlwiki from windows-1252 encoding (T128154)]] (duration: 17m 38s) |
[production] |
| 11:31 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.wdqs.restart-nginx (exit_code=0) rolling restart_daemons on A:wcqs-public |
[production] |
| 11:29 |
<jmm@cumin2002> |
START - Cookbook sre.wdqs.restart-nginx rolling restart_daemons on A:wcqs-public |
[production] |
| 11:28 |
<jmm@cumin2002> |
START - Cookbook sre.aqs.roll-restart-reboot rolling restart_daemons on A:aqs-eqiad |
[production] |