|
2022-10-25
ยง
|
| 07:08 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3315 (T321312)', diff saved to https://phabricator.wikimedia.org/P36205 and previous config saved to /var/cache/conftool/dbconfig/20221025-070837-ladsgroup.json |
[production] |
| 07:00 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1202 (re)pooling @ 1%: After upgrade', diff saved to https://phabricator.wikimedia.org/P36204 and previous config saved to /var/cache/conftool/dbconfig/20221025-070004-root.json |
[production] |
| 06:58 |
<elukey@deploy1002> |
helmfile [ml-serve-eqiad] DONE helmfile.d/admin 'sync'. |
[production] |
| 06:58 |
<elukey@deploy1002> |
helmfile [ml-serve-eqiad] START helmfile.d/admin 'sync'. |
[production] |
| 06:57 |
<elukey@deploy1002> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 06:56 |
<elukey@deploy1002> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 06:56 |
<elukey@deploy1002> |
helmfile [ml-serve-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 06:56 |
<elukey@deploy1002> |
helmfile [ml-serve-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 06:53 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2117', diff saved to https://phabricator.wikimedia.org/P36203 and previous config saved to /var/cache/conftool/dbconfig/20221025-065350-ladsgroup.json |
[production] |
| 06:53 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3315', diff saved to https://phabricator.wikimedia.org/P36202 and previous config saved to /var/cache/conftool/dbconfig/20221025-065330-ladsgroup.json |
[production] |
| 06:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2117', diff saved to https://phabricator.wikimedia.org/P36201 and previous config saved to /var/cache/conftool/dbconfig/20221025-063843-ladsgroup.json |
[production] |
| 06:38 |
<ayounsi@cumin1001> |
END (PASS) - Cookbook sre.network.peering (exit_code=0) with action 'configure' for AS: 7795 |
[production] |
| 06:38 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3315', diff saved to https://phabricator.wikimedia.org/P36200 and previous config saved to /var/cache/conftool/dbconfig/20221025-063824-ladsgroup.json |
[production] |
| 06:36 |
<ayounsi@cumin1001> |
START - Cookbook sre.network.peering with action 'configure' for AS: 7795 |
[production] |
| 06:34 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.idm.logout (exit_code=0) Logging Bruno Scarone out of all services on: 1206 hosts |
[production] |
| 06:33 |
<jmm@cumin2002> |
START - Cookbook sre.idm.logout Logging Bruno Scarone out of all services on: 1206 hosts |
[production] |
| 06:33 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.idm.logout (exit_code=0) Logging Bruno Scarone out of all services on: 799 hosts |
[production] |
| 06:32 |
<jmm@cumin2002> |
START - Cookbook sre.idm.logout Logging Bruno Scarone out of all services on: 799 hosts |
[production] |
| 06:23 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db2117 (T321312)', diff saved to https://phabricator.wikimedia.org/P36199 and previous config saved to /var/cache/conftool/dbconfig/20221025-062337-ladsgroup.json |
[production] |
| 06:23 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1096:3315 (T321312)', diff saved to https://phabricator.wikimedia.org/P36198 and previous config saved to /var/cache/conftool/dbconfig/20221025-062318-ladsgroup.json |
[production] |
| 06:17 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1096:3316 (T321312)', diff saved to https://phabricator.wikimedia.org/P36196 and previous config saved to /var/cache/conftool/dbconfig/20221025-061710-ladsgroup.json |
[production] |
| 06:16 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db2117 (T321312)', diff saved to https://phabricator.wikimedia.org/P36195 and previous config saved to /var/cache/conftool/dbconfig/20221025-061621-ladsgroup.json |
[production] |
| 06:16 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db2117.codfw.wmnet with reason: Maintenance |
[production] |
| 06:16 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db2117.codfw.wmnet with reason: Maintenance |
[production] |
| 06:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depooling db1096:3315 (T321312)', diff saved to https://phabricator.wikimedia.org/P36194 and previous config saved to /var/cache/conftool/dbconfig/20221025-061552-ladsgroup.json |
[production] |
| 06:15 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1096.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:15 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1096.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:14 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1160.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:14 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1160.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:09 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1160.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:09 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1160.eqiad.wmnet with reason: Maintenance |
[production] |
| 06:06 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Depool db1160 T321177', diff saved to https://phabricator.wikimedia.org/P36193 and previous config saved to /var/cache/conftool/dbconfig/20221025-060643-ladsgroup.json |
[production] |
| 06:01 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Promote db1138 to s4 primary and set section read-write T321177', diff saved to https://phabricator.wikimedia.org/P36192 and previous config saved to /var/cache/conftool/dbconfig/20221025-060118-ladsgroup.json |
[production] |
| 06:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set s4 eqiad as read-only for maintenance - T321177', diff saved to https://phabricator.wikimedia.org/P36191 and previous config saved to /var/cache/conftool/dbconfig/20221025-060043-ladsgroup.json |
[production] |
| 06:00 |
<Amir1> |
Starting s4 eqiad failover from db1160 to db1138 - T321177 |
[production] |
| 05:56 |
<_joe_> |
restarting pybal again on lvs1020, again for testing |
[production] |
| 05:25 |
<_joe_> |
restarting pybal on lvs1020 to test cookbook mechanism |
[production] |
| 05:19 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Set db1138 with weight 0 T321177', diff saved to https://phabricator.wikimedia.org/P36190 and previous config saved to /var/cache/conftool/dbconfig/20221025-051933-ladsgroup.json |
[production] |
| 05:19 |
<ladsgroup@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on 34 hosts with reason: Primary switchover s4 T321177 |
[production] |
| 05:18 |
<ladsgroup@cumin1001> |
START - Cookbook sre.hosts.downtime for 1:00:00 on 34 hosts with reason: Primary switchover s4 T321177 |
[production] |
| 04:15 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1194 (T321312)', diff saved to https://phabricator.wikimedia.org/P36189 and previous config saved to /var/cache/conftool/dbconfig/20221025-041558-ladsgroup.json |
[production] |
| 04:04 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 04:03 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 04:03 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 04:00 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1194', diff saved to https://phabricator.wikimedia.org/P36188 and previous config saved to /var/cache/conftool/dbconfig/20221025-040052-ladsgroup.json |
[production] |
| 03:59 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] START helmfile.d/services/mwdebug: apply |
[production] |
| 03:54 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 03:48 |
<mwdebug-deploy@deploy1002> |
helmfile [codfw] START helmfile.d/services/mwdebug: apply |
[production] |
| 03:48 |
<mwdebug-deploy@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/mwdebug: apply |
[production] |
| 03:45 |
<ladsgroup@cumin1001> |
dbctl commit (dc=all): 'Repooling after maintenance db1194', diff saved to https://phabricator.wikimedia.org/P36187 and previous config saved to /var/cache/conftool/dbconfig/20221025-034546-ladsgroup.json |
[production] |