|
2023-08-11
ยง
|
| 09:00 |
<elukey@deploy1002> |
helmfile [ml-serve-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 09:00 |
<topranks> |
depool esams site until next week for knams POP migration / rebuild |
[production] |
| 09:00 |
<elukey@deploy1002> |
helmfile [ml-serve-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 08:59 |
<elukey@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 08:59 |
<elukey@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 08:34 |
<moritzm> |
installing intel-microcode security updates on bookworm/bullseye |
[production] |
| 08:32 |
<elukey> |
expand kubelet partition on ml-serve2001 - T339231 |
[production] |
| 08:31 |
<elukey> |
restart kubelet on ml-serve1001 - T343900 |
[production] |
| 08:18 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2141.codfw.wmnet with reason: Maintenance |
[production] |
| 08:18 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2141.codfw.wmnet with reason: Maintenance |
[production] |
| 08:18 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130 (T342617)', diff saved to https://phabricator.wikimedia.org/P50472 and previous config saved to /var/cache/conftool/dbconfig/20230811-081815-ladsgroup.json |
[production] |
| 08:11 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1135 (T342617)', diff saved to https://phabricator.wikimedia.org/P50471 and previous config saved to /var/cache/conftool/dbconfig/20230811-081139-ladsgroup.json |
[production] |
| 08:11 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1135.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:11 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1135.eqiad.wmnet with reason: Maintenance |
[production] |
| 08:11 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134 (T342617)', diff saved to https://phabricator.wikimedia.org/P50470 and previous config saved to /var/cache/conftool/dbconfig/20230811-081118-ladsgroup.json |
[production] |
| 08:04 |
<elukey@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 0:30:00 on ml-serve2001.codfw.wmnet with reason: Expand the kubelet disk partition |
[production] |
| 08:04 |
<elukey@cumin1001> |
START - Cookbook sre.hosts.downtime for 0:30:00 on ml-serve2001.codfw.wmnet with reason: Expand the kubelet disk partition |
[production] |
| 08:03 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130', diff saved to https://phabricator.wikimedia.org/P50469 and previous config saved to /var/cache/conftool/dbconfig/20230811-080309-ladsgroup.json |
[production] |
| 07:57 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM rpki1001.eqiad.wmnet |
[production] |
| 07:56 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134', diff saved to https://phabricator.wikimedia.org/P50468 and previous config saved to /var/cache/conftool/dbconfig/20230811-075612-ladsgroup.json |
[production] |
| 07:54 |
<ayounsi@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM rpki1001.eqiad.wmnet |
[production] |
| 07:51 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM rpki2002.codfw.wmnet |
[production] |
| 07:48 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130', diff saved to https://phabricator.wikimedia.org/P50467 and previous config saved to /var/cache/conftool/dbconfig/20230811-074803-ladsgroup.json |
[production] |
| 07:47 |
<ayounsi@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM rpki2002.codfw.wmnet |
[production] |
| 07:41 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134', diff saved to https://phabricator.wikimedia.org/P50466 and previous config saved to /var/cache/conftool/dbconfig/20230811-074105-ladsgroup.json |
[production] |
| 07:32 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2130 (T342617)', diff saved to https://phabricator.wikimedia.org/P50465 and previous config saved to /var/cache/conftool/dbconfig/20230811-073257-ladsgroup.json |
[production] |
| 07:25 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1134 (T342617)', diff saved to https://phabricator.wikimedia.org/P50464 and previous config saved to /var/cache/conftool/dbconfig/20230811-072559-ladsgroup.json |
[production] |
| 06:12 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2179 (T342617)', diff saved to https://phabricator.wikimedia.org/P50463 and previous config saved to /var/cache/conftool/dbconfig/20230811-061250-ladsgroup.json |
[production] |
| 05:57 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2179', diff saved to https://phabricator.wikimedia.org/P50462 and previous config saved to /var/cache/conftool/dbconfig/20230811-055744-ladsgroup.json |
[production] |
| 05:46 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2130 (T342617)', diff saved to https://phabricator.wikimedia.org/P50461 and previous config saved to /var/cache/conftool/dbconfig/20230811-054649-ladsgroup.json |
[production] |
| 05:46 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2130.codfw.wmnet with reason: Maintenance |
[production] |
| 05:46 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2130.codfw.wmnet with reason: Maintenance |
[production] |
| 05:46 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116 (T342617)', diff saved to https://phabricator.wikimedia.org/P50460 and previous config saved to /var/cache/conftool/dbconfig/20230811-054628-ladsgroup.json |
[production] |
| 05:42 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2179', diff saved to https://phabricator.wikimedia.org/P50459 and previous config saved to /var/cache/conftool/dbconfig/20230811-054238-ladsgroup.json |
[production] |
| 05:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1134 (T342617)', diff saved to https://phabricator.wikimedia.org/P50458 and previous config saved to /var/cache/conftool/dbconfig/20230811-053847-ladsgroup.json |
[production] |
| 05:38 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1134.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:38 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1134.eqiad.wmnet with reason: Maintenance |
[production] |
| 05:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1132 (T342617)', diff saved to https://phabricator.wikimedia.org/P50457 and previous config saved to /var/cache/conftool/dbconfig/20230811-053826-ladsgroup.json |
[production] |
| 05:31 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116', diff saved to https://phabricator.wikimedia.org/P50456 and previous config saved to /var/cache/conftool/dbconfig/20230811-053122-ladsgroup.json |
[production] |
| 05:27 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2179 (T342617)', diff saved to https://phabricator.wikimedia.org/P50455 and previous config saved to /var/cache/conftool/dbconfig/20230811-052731-ladsgroup.json |
[production] |
| 05:23 |
<oblivian@deploy1002> |
Synchronized private/PrivateSettings.php: Adding proxy vendors (duration: 07m 33s) |
[production] |
| 05:23 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1132', diff saved to https://phabricator.wikimedia.org/P50454 and previous config saved to /var/cache/conftool/dbconfig/20230811-052320-ladsgroup.json |
[production] |
| 05:16 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116', diff saved to https://phabricator.wikimedia.org/P50453 and previous config saved to /var/cache/conftool/dbconfig/20230811-051616-ladsgroup.json |
[production] |
| 05:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1132', diff saved to https://phabricator.wikimedia.org/P50452 and previous config saved to /var/cache/conftool/dbconfig/20230811-050814-ladsgroup.json |
[production] |
| 05:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116 (T342617)', diff saved to https://phabricator.wikimedia.org/P50451 and previous config saved to /var/cache/conftool/dbconfig/20230811-050110-ladsgroup.json |
[production] |
| 04:53 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1132 (T342617)', diff saved to https://phabricator.wikimedia.org/P50450 and previous config saved to /var/cache/conftool/dbconfig/20230811-045307-ladsgroup.json |
[production] |
| 03:14 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2116 (T342617)', diff saved to https://phabricator.wikimedia.org/P50449 and previous config saved to /var/cache/conftool/dbconfig/20230811-031400-ladsgroup.json |
[production] |
| 03:13 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2116.codfw.wmnet with reason: Maintenance |
[production] |
| 03:13 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2116.codfw.wmnet with reason: Maintenance |
[production] |
| 03:13 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2112 (T342617)', diff saved to https://phabricator.wikimedia.org/P50448 and previous config saved to /var/cache/conftool/dbconfig/20230811-031339-ladsgroup.json |
[production] |