|
2023-03-08
ยง
|
| 09:45 |
<claime> |
Rebuilding production-images for 894687 |
[production] |
| 09:43 |
<jayme@deploy2002> |
helmfile [aux-k8s-eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 09:42 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116', diff saved to https://phabricator.wikimedia.org/P45437 and previous config saved to /var/cache/conftool/dbconfig/20230308-094236-marostegui.json |
[production] |
| 09:42 |
<jayme@deploy2002> |
helmfile [aux-k8s-eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 09:41 |
<jayme@deploy2002> |
helmfile [aux-k8s-eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 09:41 |
<jayme@deploy2002> |
helmfile [aux-k8s-eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 09:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2110', diff saved to https://phabricator.wikimedia.org/P45436 and previous config saved to /var/cache/conftool/dbconfig/20230308-093752-marostegui.json |
[production] |
| 09:34 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2123 (T329260)', diff saved to https://phabricator.wikimedia.org/P45435 and previous config saved to /var/cache/conftool/dbconfig/20230308-093424-marostegui.json |
[production] |
| 09:31 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2123 (T329260)', diff saved to https://phabricator.wikimedia.org/P45434 and previous config saved to /var/cache/conftool/dbconfig/20230308-093106-marostegui.json |
[production] |
| 09:31 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2123.codfw.wmnet with reason: Maintenance |
[production] |
| 09:30 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2123.codfw.wmnet with reason: Maintenance |
[production] |
| 09:30 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2111 (T329260)', diff saved to https://phabricator.wikimedia.org/P45433 and previous config saved to /var/cache/conftool/dbconfig/20230308-093045-marostegui.json |
[production] |
| 09:30 |
<moritzm> |
drain ganeti1011 for eventual reimage to Bullseye T311687 |
[production] |
| 09:27 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116', diff saved to https://phabricator.wikimedia.org/P45432 and previous config saved to /var/cache/conftool/dbconfig/20230308-092729-marostegui.json |
[production] |
| 09:22 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2110', diff saved to https://phabricator.wikimedia.org/P45431 and previous config saved to /var/cache/conftool/dbconfig/20230308-092246-marostegui.json |
[production] |
| 09:15 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2111', diff saved to https://phabricator.wikimedia.org/P45430 and previous config saved to /var/cache/conftool/dbconfig/20230308-091538-marostegui.json |
[production] |
| 09:12 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2116 (T328817)', diff saved to https://phabricator.wikimedia.org/P45429 and previous config saved to /var/cache/conftool/dbconfig/20230308-091223-marostegui.json |
[production] |
| 09:07 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2110 (T329203)', diff saved to https://phabricator.wikimedia.org/P45428 and previous config saved to /var/cache/conftool/dbconfig/20230308-090739-marostegui.json |
[production] |
| 09:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2116 (T328817)', diff saved to https://phabricator.wikimedia.org/P45426 and previous config saved to /var/cache/conftool/dbconfig/20230308-090156-marostegui.json |
[production] |
| 09:01 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2116.codfw.wmnet with reason: Maintenance |
[production] |
| 09:01 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2116.codfw.wmnet with reason: Maintenance |
[production] |
| 09:01 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2103 (T328817)', diff saved to https://phabricator.wikimedia.org/P45425 and previous config saved to /var/cache/conftool/dbconfig/20230308-090134-marostegui.json |
[production] |
| 09:00 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2111', diff saved to https://phabricator.wikimedia.org/P45424 and previous config saved to /var/cache/conftool/dbconfig/20230308-090031-marostegui.json |
[production] |
| 08:56 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2110 (T329203)', diff saved to https://phabricator.wikimedia.org/P45423 and previous config saved to /var/cache/conftool/dbconfig/20230308-085608-marostegui.json |
[production] |
| 08:56 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2110.codfw.wmnet with reason: Maintenance |
[production] |
| 08:55 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2110.codfw.wmnet with reason: Maintenance |
[production] |
| 08:55 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2106 (T329203)', diff saved to https://phabricator.wikimedia.org/P45422 and previous config saved to /var/cache/conftool/dbconfig/20230308-085546-marostegui.json |
[production] |
| 08:53 |
<akosiaris@deploy1002> |
helmfile [eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 08:53 |
<akosiaris> |
remove 10.64.64.0/21 and 10.192.64.0/21 from calico GlobalNetworkPolicies T326617 |
[production] |
| 08:52 |
<akosiaris@deploy1002> |
helmfile [eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 08:52 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2175 (re)pooling @ 100%: Repooling', diff saved to https://phabricator.wikimedia.org/P45421 and previous config saved to /var/cache/conftool/dbconfig/20230308-085159-root.json |
[production] |
| 08:50 |
<vgutierrez> |
re-enable HAProxy systemd service unit hardening in ulsfo - T323944 |
[production] |
| 08:49 |
<moritzm> |
installing git security updates |
[production] |
| 08:46 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2103', diff saved to https://phabricator.wikimedia.org/P45420 and previous config saved to /var/cache/conftool/dbconfig/20230308-084628-marostegui.json |
[production] |
| 08:45 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2111 (T329260)', diff saved to https://phabricator.wikimedia.org/P45419 and previous config saved to /var/cache/conftool/dbconfig/20230308-084525-marostegui.json |
[production] |
| 08:41 |
<marostegui> |
Deploy schema change on s3 eqiad dbmaint T329203 |
[production] |
| 08:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db2111 (T329260)', diff saved to https://phabricator.wikimedia.org/P45418 and previous config saved to /var/cache/conftool/dbconfig/20230308-084053-marostegui.json |
[production] |
| 08:40 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2111.codfw.wmnet with reason: Maintenance |
[production] |
| 08:40 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2106', diff saved to https://phabricator.wikimedia.org/P45417 and previous config saved to /var/cache/conftool/dbconfig/20230308-084040-marostegui.json |
[production] |
| 08:40 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2111.codfw.wmnet with reason: Maintenance |
[production] |
| 08:38 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repool db1113:3315', diff saved to https://phabricator.wikimedia.org/P45416 and previous config saved to /var/cache/conftool/dbconfig/20230308-083843-marostegui.json |
[production] |
| 08:38 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2101.codfw.wmnet with reason: Maintenance |
[production] |
| 08:37 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on db2101.codfw.wmnet with reason: Maintenance |
[production] |
| 08:37 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1113:3315 (re)pooling @ 15%: Repooling', diff saved to https://phabricator.wikimedia.org/P45415 and previous config saved to /var/cache/conftool/dbconfig/20230308-083731-root.json |
[production] |
| 08:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db2175 (re)pooling @ 75%: Repooling', diff saved to https://phabricator.wikimedia.org/P45414 and previous config saved to /var/cache/conftool/dbconfig/20230308-083654-root.json |
[production] |
| 08:36 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1113:3315', diff saved to https://phabricator.wikimedia.org/P45413 and previous config saved to /var/cache/conftool/dbconfig/20230308-083618-marostegui.json |
[production] |
| 08:34 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on 15 hosts with reason: Schema change |
[production] |
| 08:34 |
<marostegui> |
Deploy schema change on s3 eqiad dbmaint T329260 |
[production] |
| 08:34 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 12:00:00 on 15 hosts with reason: Schema change |
[production] |
| 08:33 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on 14 hosts with reason: Schema change |
[production] |