|
2024-04-10
ยง
|
| 18:30 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=yes; selector: name=cp1115.eqiad.wmnet,service=(cdn|ats-be) |
[production] |
| 18:28 |
<sukhe@cumin1002> |
START - Cookbook sre.hosts.reimage for host cp3071.esams.wmnet with OS bullseye |
[production] |
| 18:26 |
<sukhe@cumin1002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host cp1115.eqiad.wmnet with OS bullseye |
[production] |
| 18:24 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp3071.esams.wmnet,service=(cdn|ats-be) |
[production] |
| 18:17 |
<eevans@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 18:16 |
<eevans@deploy1002> |
helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 18:16 |
<eevans@deploy1002> |
helmfile [staging] DONE helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 18:16 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1234', diff saved to https://phabricator.wikimedia.org/P60282 and previous config saved to /var/cache/conftool/dbconfig/20240410-181618-arnaudb.json |
[production] |
| 18:15 |
<eevans@deploy1002> |
helmfile [staging] START helmfile.d/services/changeprop-jobqueue: apply |
[production] |
| 18:08 |
<sukhe@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on cp1115.eqiad.wmnet with reason: host reimage |
[production] |
| 18:05 |
<sukhe@cumin1002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on cp1115.eqiad.wmnet with reason: host reimage |
[production] |
| 18:01 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1234 (T360332)', diff saved to https://phabricator.wikimedia.org/P60281 and previous config saved to /var/cache/conftool/dbconfig/20240410-180111-arnaudb.json |
[production] |
| 17:58 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Depooling db1234 (T360332)', diff saved to https://phabricator.wikimedia.org/P60280 and previous config saved to /var/cache/conftool/dbconfig/20240410-175816-arnaudb.json |
[production] |
| 17:58 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1234.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:58 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1234.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:57 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1232 (T360332)', diff saved to https://phabricator.wikimedia.org/P60279 and previous config saved to /var/cache/conftool/dbconfig/20240410-175752-arnaudb.json |
[production] |
| 17:48 |
<sukhe@cumin1002> |
START - Cookbook sre.hosts.reimage for host cp1115.eqiad.wmnet with OS bullseye |
[production] |
| 17:48 |
<sukhe@cumin1002> |
END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host cp1115.eqiad.wmnet with OS bullseye |
[production] |
| 17:46 |
<swfrench-wmf> |
finished updating A:conf hosts to etcd-mirror 0.0.11-1 (T358636) |
[production] |
| 17:42 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1232', diff saved to https://phabricator.wikimedia.org/P60278 and previous config saved to /var/cache/conftool/dbconfig/20240410-174244-arnaudb.json |
[production] |
| 17:37 |
<sukhe@cumin1002> |
START - Cookbook sre.hosts.reimage for host cp1115.eqiad.wmnet with OS bullseye |
[production] |
| 17:37 |
<swfrench-wmf> |
restarting etcd-mirror on conf2005.codfw.wmnet for T358636 |
[production] |
| 17:35 |
<sukhe@cumin1002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts cp1115.eqiad.wmnet |
[production] |
| 17:34 |
<sukhe@cumin1002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts cp1115.eqiad.wmnet |
[production] |
| 17:27 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1232', diff saved to https://phabricator.wikimedia.org/P60277 and previous config saved to /var/cache/conftool/dbconfig/20240410-172736-arnaudb.json |
[production] |
| 17:21 |
<hashar@deploy1002> |
Finished scap: Backport for [[gerrit:1018691|TitleLibrary: Don't register external titles as dependencies (T362222)]] (duration: 18m 53s) |
[production] |
| 17:14 |
<sukhe@cumin1002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts cp1115.eqiad.wmnet |
[production] |
| 17:12 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1232 (T360332)', diff saved to https://phabricator.wikimedia.org/P60276 and previous config saved to /var/cache/conftool/dbconfig/20240410-171229-arnaudb.json |
[production] |
| 17:09 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Depooling db1232 (T360332)', diff saved to https://phabricator.wikimedia.org/P60275 and previous config saved to /var/cache/conftool/dbconfig/20240410-170930-arnaudb.json |
[production] |
| 17:09 |
<arnaudb@cumin1002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db1232.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:09 |
<arnaudb@cumin1002> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db1232.eqiad.wmnet with reason: Maintenance |
[production] |
| 17:09 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1228 (T360332)', diff saved to https://phabricator.wikimedia.org/P60274 and previous config saved to /var/cache/conftool/dbconfig/20240410-170907-arnaudb.json |
[production] |
| 17:07 |
<hashar@deploy1002> |
hashar: Continuing with sync |
[production] |
| 17:07 |
<hashar@deploy1002> |
hashar: Backport for [[gerrit:1018691|TitleLibrary: Don't register external titles as dependencies (T362222)]] synced to the testservers (https://wikitech.wikimedia.org/wiki/Mwdebug) |
[production] |
| 17:06 |
<sukhe@cumin1002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts cp1115.eqiad.wmnet |
[production] |
| 17:06 |
<sukhe@cumin1002> |
END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=0) upgrade firmware for hosts cp1115.eqiad.wmnet |
[production] |
| 17:05 |
<sukhe@cumin1002> |
START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts cp1115.eqiad.wmnet |
[production] |
| 17:05 |
<sukhe@puppetmaster1001> |
conftool action : set/pooled=no; selector: name=cp1115.eqiad.wmnet,service=(cdn|ats-be) |
[production] |
| 17:04 |
<sukhe> |
depool cp1115 for firmware downgrade for PXE boot testing: T350179 |
[production] |
| 17:04 |
<pfischer@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/cirrus-streaming-updater: apply |
[production] |
| 17:04 |
<pfischer@deploy1002> |
helmfile [eqiad] START helmfile.d/services/cirrus-streaming-updater: apply |
[production] |
| 17:04 |
<pfischer@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/cirrus-streaming-updater: apply |
[production] |
| 17:03 |
<pfischer@deploy1002> |
helmfile [eqiad] START helmfile.d/services/cirrus-streaming-updater: apply |
[production] |
| 17:02 |
<hnowlan> |
killing long-running videoscaler ffmpegs |
[production] |
| 17:02 |
<hashar@deploy1002> |
Started scap: Backport for [[gerrit:1018691|TitleLibrary: Don't register external titles as dependencies (T362222)]] |
[production] |
| 16:54 |
<arnaudb@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1228', diff saved to https://phabricator.wikimedia.org/P60272 and previous config saved to /var/cache/conftool/dbconfig/20240410-165359-arnaudb.json |
[production] |
| 16:50 |
<pfischer@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/cirrus-streaming-updater: apply |
[production] |
| 16:50 |
<pfischer@deploy1002> |
helmfile [eqiad] START helmfile.d/services/cirrus-streaming-updater: apply |
[production] |
| 16:50 |
<pfischer@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/cirrus-streaming-updater: apply |
[production] |
| 16:50 |
<pfischer@deploy1002> |
helmfile [eqiad] START helmfile.d/services/cirrus-streaming-updater: apply |
[production] |