|
2025-06-09
ยง
|
| 15:28 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1187 (T396130)', diff saved to https://phabricator.wikimedia.org/P77299 and previous config saved to /var/cache/conftool/dbconfig/20250609-152810-marostegui.json |
[production] |
| 15:28 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-search: apply |
[production] |
| 15:28 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1187.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:27 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T396130)', diff saved to https://phabricator.wikimedia.org/P77298 and previous config saved to /var/cache/conftool/dbconfig/20250609-152749-marostegui.json |
[production] |
| 15:27 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-search: apply |
[production] |
| 15:27 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-ml: apply |
[production] |
| 15:26 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1241', diff saved to https://phabricator.wikimedia.org/P77297 and previous config saved to /var/cache/conftool/dbconfig/20250609-152651-fceratto.json |
[production] |
| 15:26 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-ml: apply |
[production] |
| 15:25 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-platform-eng: apply |
[production] |
| 15:25 |
<hnowlan@deploy1003> |
helmfile [codfw] DONE helmfile.d/services/rest-gateway: apply |
[production] |
| 15:25 |
<hnowlan@deploy1003> |
helmfile [codfw] START helmfile.d/services/rest-gateway: apply |
[production] |
| 15:25 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-platform-eng: apply |
[production] |
| 15:25 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-product: apply |
[production] |
| 15:24 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-product: apply |
[production] |
| 15:24 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-main: apply |
[production] |
| 15:23 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-main: apply |
[production] |
| 15:23 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-analytics-test: apply |
[production] |
| 15:22 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-analytics-test: apply |
[production] |
| 15:22 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] DONE helmfile.d/dse-k8s-services/airflow-test-k8s: apply |
[production] |
| 15:21 |
<btullis@deploy1003> |
helmfile [dse-k8s-eqiad] START helmfile.d/dse-k8s-services/airflow-test-k8s: apply |
[production] |
| 15:16 |
<hnowlan@deploy1003> |
helmfile [staging] DONE helmfile.d/services/rest-gateway: apply |
[production] |
| 15:16 |
<hnowlan@deploy1003> |
helmfile [staging] START helmfile.d/services/rest-gateway: apply |
[production] |
| 15:12 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P77296 and previous config saved to /var/cache/conftool/dbconfig/20250609-151242-marostegui.json |
[production] |
| 15:11 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1241 (T395241)', diff saved to https://phabricator.wikimedia.org/P77295 and previous config saved to /var/cache/conftool/dbconfig/20250609-151144-fceratto.json |
[production] |
| 15:01 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db1241 (T395241)', diff saved to https://phabricator.wikimedia.org/P77294 and previous config saved to /var/cache/conftool/dbconfig/20250609-150134-fceratto.json |
[production] |
| 15:01 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1241.eqiad.wmnet with reason: Maintenance |
[production] |
| 15:01 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1238 (T395241)', diff saved to https://phabricator.wikimedia.org/P77293 and previous config saved to /var/cache/conftool/dbconfig/20250609-150108-fceratto.json |
[production] |
| 14:57 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1180', diff saved to https://phabricator.wikimedia.org/P77292 and previous config saved to /var/cache/conftool/dbconfig/20250609-145735-marostegui.json |
[production] |
| 14:46 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1238', diff saved to https://phabricator.wikimedia.org/P77291 and previous config saved to /var/cache/conftool/dbconfig/20250609-144601-fceratto.json |
[production] |
| 14:42 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1180 (T396130)', diff saved to https://phabricator.wikimedia.org/P77290 and previous config saved to /var/cache/conftool/dbconfig/20250609-144230-marostegui.json |
[production] |
| 14:40 |
<tappof@cumin1002> |
END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts prometheus7001.magru.wmnet |
[production] |
| 14:40 |
<tappof@cumin1002> |
END (PASS) - Cookbook sre.dns.netbox (exit_code=0) |
[production] |
| 14:40 |
<tappof@cumin1002> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: prometheus7001.magru.wmnet decommissioned, removing all IPs except the asset tag one - tappof@cumin1002" |
[production] |
| 14:39 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Depooling db1180 (T396130)', diff saved to https://phabricator.wikimedia.org/P77289 and previous config saved to /var/cache/conftool/dbconfig/20250609-143938-marostegui.json |
[production] |
| 14:39 |
<marostegui@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1180.eqiad.wmnet with reason: Maintenance |
[production] |
| 14:39 |
<tappof@cumin1002> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: prometheus7001.magru.wmnet decommissioned, removing all IPs except the asset tag one - tappof@cumin1002" |
[production] |
| 14:39 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1168 (T396130)', diff saved to https://phabricator.wikimedia.org/P77288 and previous config saved to /var/cache/conftool/dbconfig/20250609-143917-marostegui.json |
[production] |
| 14:36 |
<tappof@cumin1002> |
START - Cookbook sre.dns.netbox |
[production] |
| 14:31 |
<tappof@cumin1002> |
START - Cookbook sre.hosts.decommission for hosts prometheus7001.magru.wmnet |
[production] |
| 14:30 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1238', diff saved to https://phabricator.wikimedia.org/P77287 and previous config saved to /var/cache/conftool/dbconfig/20250609-143054-fceratto.json |
[production] |
| 14:30 |
<fnegri@cumin1002> |
conftool action : set/pooled=yes; selector: name=clouddb1015.eqiad.wmnet |
[production] |
| 14:24 |
<hnowlan@deploy1003> |
helmfile [staging] DONE helmfile.d/services/rest-gateway: apply |
[production] |
| 14:24 |
<hnowlan@deploy1003> |
helmfile [staging] START helmfile.d/services/rest-gateway: apply |
[production] |
| 14:24 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P77286 and previous config saved to /var/cache/conftool/dbconfig/20250609-142410-marostegui.json |
[production] |
| 14:18 |
<godog> |
rollout cgroup memory limit + gomemlimit for thanos-sidecar - T394318 |
[production] |
| 14:15 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1238 (T395241)', diff saved to https://phabricator.wikimedia.org/P77285 and previous config saved to /var/cache/conftool/dbconfig/20250609-141548-fceratto.json |
[production] |
| 14:15 |
<fceratto@deploy1003> |
helmfile [aux-k8s-eqiad] 'sync' command on namespace 'zarcillo' for release 'main' . |
[production] |
| 14:09 |
<marostegui@cumin1002> |
dbctl commit (dc=all): 'Repooling after maintenance db1168', diff saved to https://phabricator.wikimedia.org/P77284 and previous config saved to /var/cache/conftool/dbconfig/20250609-140903-marostegui.json |
[production] |
| 14:07 |
<fceratto@cumin1002> |
dbctl commit (dc=all): 'Depooling db1238 (T395241)', diff saved to https://phabricator.wikimedia.org/P77283 and previous config saved to /var/cache/conftool/dbconfig/20250609-140722-fceratto.json |
[production] |
| 14:07 |
<fceratto@cumin1002> |
DONE (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1238.eqiad.wmnet with reason: Maintenance |
[production] |