|
2026-04-17
§
|
| 14:13 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 14:12 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 14:12 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 14:12 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1233 (T419961)', diff saved to https://phabricator.wikimedia.org/P91073 and previous config saved to /var/cache/conftool/dbconfig/20260417-141222-fceratto.json |
[production] |
| 14:12 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 14:11 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 14:11 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2168', diff saved to https://phabricator.wikimedia.org/P91072 and previous config saved to /var/cache/conftool/dbconfig/20260417-141123-fceratto.json |
[production] |
| 14:10 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 14:09 |
<urandom> |
decommissioning Cassandra, aqs1011 [a,b] — T412830 |
[production] |
| 14:06 |
<fabfur@cumin1003> |
END (PASS) - Cookbook sre.cdn.roll-restart-varnish (exit_code=0) rolling restart of Varnish on 1 hosts matching query P{cp3073.*} |
[production] |
| 14:06 |
<eevans@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 30 days, 0:00:00 on aqs1011.eqiad.wmnet with reason: Bootstrapping — T412830 |
[production] |
| 14:05 |
<fabfur@cumin1003> |
START - Cookbook sre.cdn.roll-restart-varnish rolling restart of Varnish on 1 hosts matching query P{cp3073.*} |
[production] |
| 14:04 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Depooling db1233 (T419961)', diff saved to https://phabricator.wikimedia.org/P91071 and previous config saved to /var/cache/conftool/dbconfig/20260417-140454-fceratto.json |
[production] |
| 14:04 |
<fceratto@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1233.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:04 |
<fabfur@cumin1003> |
END (PASS) - Cookbook sre.cdn.roll-restart-varnish (exit_code=0) rolling restart of Varnish on 1 hosts matching query P{cp3072.*} |
[production] |
| 14:04 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1229 (T419961)', diff saved to https://phabricator.wikimedia.org/P91070 and previous config saved to /var/cache/conftool/dbconfig/20260417-140424-fceratto.json |
[production] |
| 14:04 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 14:03 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 14:03 |
<fabfur@cumin1003> |
START - Cookbook sre.cdn.roll-restart-varnish rolling restart of Varnish on 1 hosts matching query P{cp3072.*} |
[production] |
| 14:02 |
<fabfur@cumin1003> |
END (PASS) - Cookbook sre.cdn.roll-restart-varnish (exit_code=0) rolling restart of Varnish on 1 hosts matching query P{cp3070.*} |
[production] |
| 14:01 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] DONE helmfile.d/admin 'sync'. |
[production] |
| 14:01 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2168 (T419635)', diff saved to https://phabricator.wikimedia.org/P91069 and previous config saved to /var/cache/conftool/dbconfig/20260417-140115-fceratto.json |
[production] |
| 14:01 |
<fabfur@cumin1003> |
START - Cookbook sre.cdn.roll-restart-varnish rolling restart of Varnish on 1 hosts matching query P{cp3070.*} |
[production] |
| 14:00 |
<fabfur@cumin1003> |
END (PASS) - Cookbook sre.cdn.roll-restart-varnish (exit_code=0) rolling restart of Varnish on 1 hosts matching query P{cp3069.*} |
[production] |
| 14:00 |
<fabfur> |
restart varnish on cp3069, cp3070, cp3072, cp3073 to clear alerts |
[production] |
| 14:00 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Depooling db2168 (T419635)', diff saved to https://phabricator.wikimedia.org/P91068 and previous config saved to /var/cache/conftool/dbconfig/20260417-140003-fceratto.json |
[production] |
| 13:59 |
<fceratto@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2168.codfw.wmnet with reason: Maintenance |
[production] |
| 13:59 |
<dpogorzelski@deploy1003> |
helmfile [ml-staging-codfw] START helmfile.d/admin 'sync'. |
[production] |
| 13:59 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2159 (T419635)', diff saved to https://phabricator.wikimedia.org/P91067 and previous config saved to /var/cache/conftool/dbconfig/20260417-135938-fceratto.json |
[production] |
| 13:58 |
<fabfur@cumin1003> |
START - Cookbook sre.cdn.roll-restart-varnish rolling restart of Varnish on 1 hosts matching query P{cp3069.*} |
[production] |
| 13:57 |
<fabfur@cumin1003> |
END (PASS) - Cookbook sre.cdn.roll-restart-varnish (exit_code=0) rolling restart of Varnish on 1 hosts matching query P{cp3066.*} |
[production] |
| 13:54 |
<fabfur@cumin1003> |
START - Cookbook sre.cdn.roll-restart-varnish rolling restart of Varnish on 1 hosts matching query P{cp3066.*} |
[production] |
| 13:54 |
<fabfur> |
restarting varnish on cp3066 to clear alerts |
[production] |
| 13:54 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1229', diff saved to https://phabricator.wikimedia.org/P91066 and previous config saved to /var/cache/conftool/dbconfig/20260417-135416-fceratto.json |
[production] |
| 13:52 |
<otto@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/mw-page-html-content-change-enrich: apply |
[production] |
| 13:52 |
<otto@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/mw-page-html-content-change-enrich: apply |
[production] |
| 13:49 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2159', diff saved to https://phabricator.wikimedia.org/P91065 and previous config saved to /var/cache/conftool/dbconfig/20260417-134930-fceratto.json |
[production] |
| 13:44 |
<jmm@dns1004> |
END - running authdns-update |
[production] |
| 13:44 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1229', diff saved to https://phabricator.wikimedia.org/P91064 and previous config saved to /var/cache/conftool/dbconfig/20260417-134408-fceratto.json |
[production] |
| 13:43 |
<jmm@dns1004> |
START - running authdns-update |
[production] |
| 13:42 |
<inflatador> |
bking@apt1002 sudo -E reprepro -C component/opensearch2 include trixie-wikimedia /home/bking/wmf-opensearch-search-plugins-2.19.5+5-trixie/wmf-opensearch-search-plugins_2.19.5+5_amd64.changes |
[production] |
| 13:39 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2159', diff saved to https://phabricator.wikimedia.org/P91063 and previous config saved to /var/cache/conftool/dbconfig/20260417-133923-fceratto.json |
[production] |
| 13:34 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db1229 (T419961)', diff saved to https://phabricator.wikimedia.org/P91062 and previous config saved to /var/cache/conftool/dbconfig/20260417-133359-fceratto.json |
[production] |
| 13:29 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2159 (T419635)', diff saved to https://phabricator.wikimedia.org/P91061 and previous config saved to /var/cache/conftool/dbconfig/20260417-132914-fceratto.json |
[production] |
| 13:28 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Depooling db2159 (T419635)', diff saved to https://phabricator.wikimedia.org/P91060 and previous config saved to /var/cache/conftool/dbconfig/20260417-132802-fceratto.json |
[production] |
| 13:27 |
<fceratto@cumin1003> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 12:00:00 on db2159.codfw.wmnet with reason: Maintenance |
[production] |
| 13:27 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Repooling after maintenance db2150 (T419635)', diff saved to https://phabricator.wikimedia.org/P91059 and previous config saved to /var/cache/conftool/dbconfig/20260417-132738-fceratto.json |
[production] |
| 13:27 |
<jiji@deploy1003> |
helmfile [codfw] DONE helmfile.d/services/mw-parsoid: apply |
[production] |
| 13:26 |
<jiji@deploy1003> |
helmfile [codfw] START helmfile.d/services/mw-parsoid: apply |
[production] |
| 13:26 |
<fceratto@cumin1003> |
dbctl commit (dc=all): 'Depooling db1229 (T419961)', diff saved to https://phabricator.wikimedia.org/P91058 and previous config saved to /var/cache/conftool/dbconfig/20260417-132628-fceratto.json |
[production] |