|
2022-11-29
ยง
|
| 12:51 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:51 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1202 (T323907)', diff saved to https://phabricator.wikimedia.org/P41710 and previous config saved to /var/cache/conftool/dbconfig/20221129-125121-ladsgroup.json |
[production] |
| 12:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1132', diff saved to https://phabricator.wikimedia.org/P41709 and previous config saved to /var/cache/conftool/dbconfig/20221129-124203-marostegui.json |
[production] |
| 12:36 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1202', diff saved to https://phabricator.wikimedia.org/P41708 and previous config saved to /var/cache/conftool/dbconfig/20221129-123614-ladsgroup.json |
[production] |
| 12:31 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2122 (T323907)', diff saved to https://phabricator.wikimedia.org/P41707 and previous config saved to /var/cache/conftool/dbconfig/20221129-123134-ladsgroup.json |
[production] |
| 12:31 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2122.codfw.wmnet with reason: Maintenance |
[production] |
| 12:31 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2122.codfw.wmnet with reason: Maintenance |
[production] |
| 12:31 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2121 (T323907)', diff saved to https://phabricator.wikimedia.org/P41706 and previous config saved to /var/cache/conftool/dbconfig/20221129-123113-ladsgroup.json |
[production] |
| 12:26 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1132 (T321126)', diff saved to https://phabricator.wikimedia.org/P41705 and previous config saved to /var/cache/conftool/dbconfig/20221129-122657-marostegui.json |
[production] |
| 12:24 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1132 (T321126)', diff saved to https://phabricator.wikimedia.org/P41704 and previous config saved to /var/cache/conftool/dbconfig/20221129-122436-marostegui.json |
[production] |
| 12:24 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1132.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:24 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1132.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:24 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1128 (T321126)', diff saved to https://phabricator.wikimedia.org/P41703 and previous config saved to /var/cache/conftool/dbconfig/20221129-122414-marostegui.json |
[production] |
| 12:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1202', diff saved to https://phabricator.wikimedia.org/P41702 and previous config saved to /var/cache/conftool/dbconfig/20221129-122108-ladsgroup.json |
[production] |
| 12:16 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2121', diff saved to https://phabricator.wikimedia.org/P41701 and previous config saved to /var/cache/conftool/dbconfig/20221129-121606-ladsgroup.json |
[production] |
| 12:13 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2177 (T322618)', diff saved to https://phabricator.wikimedia.org/P41700 and previous config saved to /var/cache/conftool/dbconfig/20221129-121354-ladsgroup.json |
[production] |
| 12:09 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1128', diff saved to https://phabricator.wikimedia.org/P41699 and previous config saved to /var/cache/conftool/dbconfig/20221129-120907-marostegui.json |
[production] |
| 12:06 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1202 (T323907)', diff saved to https://phabricator.wikimedia.org/P41698 and previous config saved to /var/cache/conftool/dbconfig/20221129-120601-ladsgroup.json |
[production] |
| 12:05 |
<hnowlan@deploy1002> |
helmfile [staging] DONE helmfile.d/services/thumbor: sync |
[production] |
| 12:04 |
<hnowlan@deploy1002> |
helmfile [staging] START helmfile.d/services/thumbor: sync |
[production] |
| 12:03 |
<hnowlan@deploy1002> |
helmfile [staging] DONE helmfile.d/services/thumbor: sync |
[production] |
| 12:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2121', diff saved to https://phabricator.wikimedia.org/P41697 and previous config saved to /var/cache/conftool/dbconfig/20221129-120100-ladsgroup.json |
[production] |
| 11:58 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2177', diff saved to https://phabricator.wikimedia.org/P41696 and previous config saved to /var/cache/conftool/dbconfig/20221129-115847-ladsgroup.json |
[production] |
| 11:54 |
<filippo@cumin1001> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host grafana2001.codfw.wmnet |
[production] |
| 11:54 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1128', diff saved to https://phabricator.wikimedia.org/P41695 and previous config saved to /var/cache/conftool/dbconfig/20221129-115401-marostegui.json |
[production] |
| 11:53 |
<hnowlan@deploy1002> |
helmfile [staging] START helmfile.d/services/thumbor: sync |
[production] |
| 11:53 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host puppetdb2003.codfw.wmnet |
[production] |
| 11:47 |
<filippo@cumin1001> |
START - Cookbook sre.hosts.reboot-single for host grafana2001.codfw.wmnet |
[production] |
| 11:47 |
<marostegui> |
Drop scholarships database from m2 T243037 |
[production] |
| 11:47 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reboot-single for host puppetdb2003.codfw.wmnet |
[production] |
| 11:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2121 (T323907)', diff saved to https://phabricator.wikimedia.org/P41694 and previous config saved to /var/cache/conftool/dbconfig/20221129-114553-ladsgroup.json |
[production] |
| 11:43 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2177', diff saved to https://phabricator.wikimedia.org/P41693 and previous config saved to /var/cache/conftool/dbconfig/20221129-114341-ladsgroup.json |
[production] |
| 11:43 |
<godog> |
+100G to global/prometheus in eqiad |
[production] |
| 11:38 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1128 (T321126)', diff saved to https://phabricator.wikimedia.org/P41692 and previous config saved to /var/cache/conftool/dbconfig/20221129-113854-marostegui.json |
[production] |
| 11:37 |
<moritzm> |
uploaded ferm 2.5.1-1.1+wmf11u1 to apt.wikimedia.org/bookworm (rebasing our systemd startup fixes to what's in bookworm) T321783 |
[production] |
| 11:37 |
<hnowlan@deploy1002> |
helmfile [staging] DONE helmfile.d/services/thumbor: sync |
[production] |
| 11:37 |
<hnowlan@deploy1002> |
helmfile [staging] START helmfile.d/services/thumbor: sync |
[production] |
| 11:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1128 (T321126)', diff saved to https://phabricator.wikimedia.org/P41691 and previous config saved to /var/cache/conftool/dbconfig/20221129-113633-marostegui.json |
[production] |
| 11:36 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 5:00:00 on db1128.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:36 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 5:00:00 on db1128.eqiad.wmnet with reason: Maintenance |
[production] |
| 11:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1119 (T321126)', diff saved to https://phabricator.wikimedia.org/P41690 and previous config saved to /var/cache/conftool/dbconfig/20221129-113612-marostegui.json |
[production] |
| 11:34 |
<hnowlan@deploy1002> |
helmfile [staging] DONE helmfile.d/services/thumbor: sync |
[production] |
| 11:34 |
<hnowlan@deploy1002> |
helmfile [staging] START helmfile.d/services/thumbor: sync |
[production] |
| 11:28 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2177 (T322618)', diff saved to https://phabricator.wikimedia.org/P41689 and previous config saved to /var/cache/conftool/dbconfig/20221129-112835-ladsgroup.json |
[production] |
| 11:21 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1119', diff saved to https://phabricator.wikimedia.org/P41688 and previous config saved to /var/cache/conftool/dbconfig/20221129-112106-marostegui.json |
[production] |
| 11:21 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2177 (T322618)', diff saved to https://phabricator.wikimedia.org/P41687 and previous config saved to /var/cache/conftool/dbconfig/20221129-112053-ladsgroup.json |
[production] |
| 11:20 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db2177.codfw.wmnet with reason: Maintenance |
[production] |
| 11:20 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db2177.codfw.wmnet with reason: Maintenance |
[production] |
| 11:20 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2156 (T322618)', diff saved to https://phabricator.wikimedia.org/P41686 and previous config saved to /var/cache/conftool/dbconfig/20221129-112043-ladsgroup.json |
[production] |
| 11:10 |
<oblivian@cumin1001> |
END (PASS) - Cookbook sre.hosts.remove-downtime (exit_code=0) for 42 hosts |
[production] |