|
2023-05-03
§
|
| 15:07 |
<eevans@cumin1001> |
START - Cookbook sre.cassandra.roll-restart for nodes matching restbase1016.eqiad.wmnet: Upgrade Cassandra — T335383 - eevans@cumin1001 |
[production] |
| 15:03 |
<eevans@cumin1001> |
END (PASS) - Cookbook sre.cassandra.roll-restart (exit_code=0) for nodes matching restbase2012.codfw.wmnet: Upgrade Cassandra — T335383 - eevans@cumin1001 |
[production] |
| 15:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2171:3316 (T335838)', diff saved to https://phabricator.wikimedia.org/P47411 and previous config saved to /var/cache/conftool/dbconfig/20230503-150103-ladsgroup.json |
[production] |
| 15:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2171:3315 (T335838)', diff saved to https://phabricator.wikimedia.org/P47410 and previous config saved to /var/cache/conftool/dbconfig/20230503-150042-ladsgroup.json |
[production] |
| 15:00 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2171.codfw.wmnet with reason: Maintenance |
[production] |
| 15:00 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2171.codfw.wmnet with reason: Maintenance |
[production] |
| 15:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317 (T335838)', diff saved to https://phabricator.wikimedia.org/P47409 and previous config saved to /var/cache/conftool/dbconfig/20230503-150017-ladsgroup.json |
[production] |
| 14:59 |
<sukhe> |
fix backup route for high-traffic2 in codfw: set routing-options static route 208.80.153.240/28 next-hop 10.192.17.7 |
[production] |
| 14:54 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2157', diff saved to https://phabricator.wikimedia.org/P47408 and previous config saved to /var/cache/conftool/dbconfig/20230503-145440-ladsgroup.json |
[production] |
| 14:54 |
<sukhe> |
[finished] homer "cr*-codfw*" commit "Gerrit: 914344 remove decommissioned host lvs2007": T335777 |
[production] |
| 14:53 |
<eevans@cumin1001> |
START - Cookbook sre.cassandra.roll-restart for nodes matching restbase2012.codfw.wmnet: Upgrade Cassandra — T335383 - eevans@cumin1001 |
[production] |
| 14:52 |
<sukhe> |
homer "cr*-codfw*" commit "Gerrit: 914344 remove decommissioned host lvs2007": T335777 |
[production] |
| 14:46 |
<sukhe@cumin2002> |
END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts lvs2007.codfw.wmnet |
[production] |
| 14:46 |
<sukhe@cumin2002> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 14:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317', diff saved to https://phabricator.wikimedia.org/P47407 and previous config saved to /var/cache/conftool/dbconfig/20230503-144511-ladsgroup.json |
[production] |
| 14:45 |
<sukhe@cumin2002> |
START - Cookbook sre.dns.netbox |
[production] |
| 14:43 |
<slyngshede@cumin1001> |
END (PASS) - Cookbook sre.ganeti.reboot-vm (exit_code=0) for VM idm1001.wikimedia.org |
[production] |
| 14:42 |
<ottomata> |
Grant IdempotentWrite Kafka Cluster ACL to User:ANONYOUS in kafka logging clusters - T334733 |
[production] |
| 14:40 |
<jclark@cumin1001> |
END (FAIL) - Cookbook sre.dns.netbox (exit_code=99) |
[production] |
| 14:40 |
<jclark@cumin1001> |
END (FAIL) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=99) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt frav1003 - jclark@cumin1001" |
[production] |
| 14:40 |
<slyngshede@cumin1001> |
START - Cookbook sre.ganeti.reboot-vm for VM idm1001.wikimedia.org |
[production] |
| 14:39 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2157', diff saved to https://phabricator.wikimedia.org/P47406 and previous config saved to /var/cache/conftool/dbconfig/20230503-143933-ladsgroup.json |
[production] |
| 14:38 |
<jclark@cumin1001> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: update mgmt frav1003 - jclark@cumin1001" |
[production] |
| 14:36 |
<jclark@cumin1001> |
START - Cookbook sre.dns.netbox |
[production] |
| 14:36 |
<sukhe@cumin2002> |
START - Cookbook sre.hosts.decommission for hosts lvs2007.codfw.wmnet |
[production] |
| 14:33 |
<sukhe> |
set routing-options static route 208.80.153.224/28 next-hop 10.192.49.7 [move static route for high-traffic1 to lvs2010]: T335777 |
[production] |
| 14:30 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317', diff saved to https://phabricator.wikimedia.org/P47405 and previous config saved to /var/cache/conftool/dbconfig/20230503-143005-ladsgroup.json |
[production] |
| 14:26 |
<ottomata> |
Grant IdempotentWrite Kafka Cluster ACL to User:ANONYOUS in kafka main clusters - T334733 |
[production] |
| 14:24 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2157 (T335838)', diff saved to https://phabricator.wikimedia.org/P47404 and previous config saved to /var/cache/conftool/dbconfig/20230503-142427-ladsgroup.json |
[production] |
| 14:18 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2157 (T335838)', diff saved to https://phabricator.wikimedia.org/P47403 and previous config saved to /var/cache/conftool/dbconfig/20230503-141817-ladsgroup.json |
[production] |
| 14:18 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2157.codfw.wmnet with reason: Maintenance |
[production] |
| 14:17 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2157.codfw.wmnet with reason: Maintenance |
[production] |
| 14:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2137:3315 (T335838)', diff saved to https://phabricator.wikimedia.org/P47402 and previous config saved to /var/cache/conftool/dbconfig/20230503-141752-ladsgroup.json |
[production] |
| 14:16 |
<sukhe@deploy1002> |
Locking from deployment [ALL REPOSITORIES]: LVS reimaging in codfw, blocking deploys T326767 |
[production] |
| 14:16 |
<otto@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/eventgate-main: apply |
[production] |
| 14:15 |
<otto@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventgate-main: apply |
[production] |
| 14:15 |
<otto@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/eventgate-main: apply |
[production] |
| 14:15 |
<otto@deploy1002> |
helmfile [codfw] START helmfile.d/services/eventgate-main: apply |
[production] |
| 14:15 |
<otto@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/eventgate-analytics: apply |
[production] |
| 14:14 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2169:3317 (T335838)', diff saved to https://phabricator.wikimedia.org/P47401 and previous config saved to /var/cache/conftool/dbconfig/20230503-141458-ladsgroup.json |
[production] |
| 14:14 |
<otto@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventgate-analytics: apply |
[production] |
| 14:13 |
<otto@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/eventgate-analytics: apply |
[production] |
| 14:13 |
<otto@deploy1002> |
helmfile [staging] DONE helmfile.d/services/eventgate-main: apply |
[production] |
| 14:13 |
<otto@deploy1002> |
helmfile [staging] START helmfile.d/services/eventgate-main: apply |
[production] |
| 14:13 |
<otto@deploy1002> |
helmfile [codfw] START helmfile.d/services/eventgate-analytics: apply |
[production] |
| 14:12 |
<otto@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/eventgate-analytics-external: apply |
[production] |
| 14:12 |
<otto@deploy1002> |
helmfile [eqiad] START helmfile.d/services/eventgate-analytics-external: apply |
[production] |
| 14:12 |
<otto@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/eventgate-analytics-external: apply |
[production] |
| 14:12 |
<otto@deploy1002> |
helmfile [staging] DONE helmfile.d/services/eventgate-analytics: apply |
[production] |
| 14:11 |
<otto@deploy1002> |
helmfile [staging] START helmfile.d/services/eventgate-analytics: apply |
[production] |