|
2025-06-09
ยง
|
| 15:25 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-product: apply |
[production] |
| 15:24 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-product: apply |
[production] |
| 15:24 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-main: apply |
[production] |
| 15:23 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-main: apply |
[production] |
| 15:23 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply |
[production] |
| 15:22 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply |
[production] |
| 15:22 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply |
[production] |
| 15:21 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply |
[production] |
| 15:16 |
<hnowlan@deploy1003> |
helmfile [staging] DONE helmfile.d/services/rest-gateway: apply |
[production] |
| 15:16 |
<hnowlan@deploy1003> |
helmfile [staging] START helmfile.d/services/rest-gateway: apply |
[production] |
| 15:12 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P77296 and previous config saved to /var/cache/conftool/dbconfig/20250609-151242-marostegui.json |
[production] |
| 15:11 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1241 (T395241)', diff saved to https://phabricator.wikimedia.org/P77295 and previous config saved to /var/cache/conftool/dbconfig/20250609-151144-fceratto.json |
[production] |
| 15:01 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db1241 (T395241)', diff saved to https://phabricator.wikimedia.org/P77294 and previous config saved to /var/cache/conftool/dbconfig/20250609-150134-fceratto.json |
[production] |
| 15:01 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1241.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:01 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1238 (T395241)', diff saved to https://phabricator.wikimedia.org/P77293 and previous config saved to /var/cache/conftool/dbconfig/20250609-150108-fceratto.json |
[production] |
| 14:57 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P77292 and previous config saved to /var/cache/conftool/dbconfig/20250609-145735-marostegui.json |
[production] |
| 14:46 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1238', diff saved to https://phabricator.wikimedia.org/P77291 and previous config saved to /var/cache/conftool/dbconfig/20250609-144601-fceratto.json |
[production] |
| 14:42 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T396130)', diff saved to https://phabricator.wikimedia.org/P77290 and previous config saved to /var/cache/conftool/dbconfig/20250609-144230-marostegui.json |
[production] |
| 14:40 |
<tappof@cumin1002> |
END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts prometheus7001.magru.wmnet |
[production] |
| 14:40 |
<tappof@cumin1002> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 14:40 |
<tappof@cumin1002> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: prometheus7001.magru.wmnet decommissioned, removing all IPs except the asset tag one - tappof@cumin1002" |
[production] |
| 14:39 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1180 (T396130)', diff saved to https://phabricator.wikimedia.org/P77289 and previous config saved to /var/cache/conftool/dbconfig/20250609-143938-marostegui.json |
[production] |
| 14:39 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1180.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:39 |
<tappof@cumin1002> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: prometheus7001.magru.wmnet decommissioned, removing all IPs except the asset tag one - tappof@cumin1002" |
[production] |
| 14:39 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T396130)', diff saved to https://phabricator.wikimedia.org/P77288 and previous config saved to /var/cache/conftool/dbconfig/20250609-143917-marostegui.json |
[production] |
| 14:36 |
<tappof@cumin1002> |
START - Cookbook sre.dns.netbox |
[production] |
| 14:31 |
<tappof@cumin1002> |
START - Cookbook sre.hosts.decommission for hosts prometheus7001.magru.wmnet |
[production] |
| 14:30 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1238', diff saved to https://phabricator.wikimedia.org/P77287 and previous config saved to /var/cache/conftool/dbconfig/20250609-143054-fceratto.json |
[production] |
| 14:30 |
<fnegri@cumin1002> |
conftool action : set/pooled=yes; selector: name=clouddb1015.eqiad.wmnet |
[production] |
| 14:24 |
<hnowlan@deploy1003> |
helmfile [staging] DONE helmfile.d/services/rest-gateway: apply |
[production] |
| 14:24 |
<hnowlan@deploy1003> |
helmfile [staging] START helmfile.d/services/rest-gateway: apply |
[production] |
| 14:24 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P77286 and previous config saved to /var/cache/conftool/dbconfig/20250609-142410-marostegui.json |
[production] |
| 14:18 |
<godog> |
rollout cgroup memory limit + gomemlimit for thanos-sidecar - T394318 |
[production] |
| 14:15 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1238 (T395241)', diff saved to https://phabricator.wikimedia.org/P77285 and previous config saved to /var/cache/conftool/dbconfig/20250609-141548-fceratto.json |
[production] |
| 14:15 |
<fceratto@deploy1003> |
helmfile [aux-k8s-eqiad] 'sync' command on namespace 'zarcillo' for release 'main' . |
[production] |
| 14:09 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P77284 and previous config saved to /var/cache/conftool/dbconfig/20250609-140903-marostegui.json |
[production] |
| 14:07 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db1238 (T395241)', diff saved to https://phabricator.wikimedia.org/P77283 and previous config saved to /var/cache/conftool/dbconfig/20250609-140722-fceratto.json |
[production] |
| 14:07 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1238.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:06 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1221 (T395241)', diff saved to https://phabricator.wikimedia.org/P77282 and previous config saved to /var/cache/conftool/dbconfig/20250609-140656-fceratto.json |
[production] |
| 13:55 |
<sukhe@dns1004> |
END - running authdns-update |
[production] |
| 13:55 |
<sukhe@dns1004> |
START - running authdns-update |
[production] |
| 13:53 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T396130)', diff saved to https://phabricator.wikimedia.org/P77281 and previous config saved to /var/cache/conftool/dbconfig/20250609-135355-marostegui.json |
[production] |
| 13:52 |
<fnegri@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on clouddb1015.eqiad.wmnet with reason: Upgrading clouddbs T394372 |
[production] |
| 13:51 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1221', diff saved to https://phabricator.wikimedia.org/P77280 and previous config saved to /var/cache/conftool/dbconfig/20250609-135150-fceratto.json |
[production] |
| 13:51 |
<fnegri@cumin1002> |
conftool action : set/pooled=no; selector: name=clouddb1015.eqiad.wmnet |
[production] |
| 13:51 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1168 (T396130)', diff saved to https://phabricator.wikimedia.org/P77279 and previous config saved to /var/cache/conftool/dbconfig/20250609-135105-marostegui.json |
[production] |
| 13:50 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1168.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:50 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1165 (T396130)', diff saved to https://phabricator.wikimedia.org/P77278 and previous config saved to /var/cache/conftool/dbconfig/20250609-135043-marostegui.json |
[production] |
| 13:45 |
<sukhe@cumin1002> |
END (PASS) - Cookbook sre.dns.roll-reboot (exit_code=0) rolling reboot on P{dns5004*} and (A:dnsbox) |
[production] |
| 13:45 |
<sukhe@cumin1002> |
cookbooks.sre.dns.roll-reboot finished rebooting dns5004.wikimedia.org |
[production] |