|
2022-02-11
ยง
|
| 19:13 |
<mutante> |
running puppet on all ores machines to install aspell-hi (gerrit:761974) which for some reason was installed on a random subset of ores servers (1002,2001,2005 but not the other 19 ones) T300195 T252581 - after this the package is now installed on 18 servers (1001-1009, 2001-2009) |
[production] |
| 16:54 |
<hnowlan@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 16:54 |
<hnowlan@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: sync on staging |
[production] |
| 16:54 |
<hnowlan@deploy1002> |
helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 16:53 |
<hnowlan@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 16:53 |
<hnowlan@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: sync on staging |
[production] |
| 16:53 |
<hnowlan@deploy1002> |
helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 16:32 |
<btullis@cumin1001> |
END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host datahubsearch1001.eqiad.wmnet |
[production] |
| 16:13 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1113:3315 (T300775)', diff saved to https://phabricator.wikimedia.org/P20611 and previous config saved to /var/cache/conftool/dbconfig/20220211-161324-marostegui.json |
[production] |
| 16:13 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 16:13 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on db1113.eqiad.wmnet with reason: Maintenance |
[production] |
| 16:03 |
<btullis@cumin1001> |
START - Cookbook sre.ganeti.makevm for new host datahubsearch1001.eqiad.wmnet |
[production] |
| 14:23 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.decommission (exit_code=0) for hosts auth2001.codfw.wmnet |
[production] |
| 14:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1098:3316 (re)pooling @ 100%: repooling after schema change', diff saved to https://phabricator.wikimedia.org/P20610 and previous config saved to /var/cache/conftool/dbconfig/20220211-142045-root.json |
[production] |
| 14:07 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.decommission for hosts auth2001.codfw.wmnet |
[production] |
| 14:05 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1098:3316 (re)pooling @ 75%: repooling after schema change', diff saved to https://phabricator.wikimedia.org/P20609 and previous config saved to /var/cache/conftool/dbconfig/20220211-140540-root.json |
[production] |
| 13:50 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1098:3316 (re)pooling @ 50%: repooling after schema change', diff saved to https://phabricator.wikimedia.org/P20608 and previous config saved to /var/cache/conftool/dbconfig/20220211-135037-root.json |
[production] |
| 13:35 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1098:3316 (re)pooling @ 25%: repooling after schema change', diff saved to https://phabricator.wikimedia.org/P20607 and previous config saved to /var/cache/conftool/dbconfig/20220211-133533-root.json |
[production] |
| 13:20 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'db1098:3316 (re)pooling @ 10%: repooling after schema change', diff saved to https://phabricator.wikimedia.org/P20606 and previous config saved to /var/cache/conftool/dbconfig/20220211-132028-root.json |
[production] |
| 13:19 |
<jmm@cumin2002> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host ganeti1011.eqiad.wmnet with OS buster |
[production] |
| 13:18 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:18 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:17 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:17 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:15 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Depooling db1098:3316 (T300662)', diff saved to https://phabricator.wikimedia.org/P20605 and previous config saved to /var/cache/conftool/dbconfig/20220211-131507-marostegui.json |
[production] |
| 13:15 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 13:15 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 6:00:00 on db1098.eqiad.wmnet with reason: Maintenance |
[production] |
| 12:53 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reimage for host ganeti1011.eqiad.wmnet with OS buster |
[production] |
| 12:41 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ganeti1016.eqiad.wmnet with OS buster |
[production] |
| 12:13 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reimage for host ganeti1016.eqiad.wmnet with OS buster |
[production] |
| 10:43 |
<hnowlan@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 10:42 |
<hnowlan@deploy1002> |
helmfile [eqiad] DONE helmfile.d/services/changeprop-jobqueue: sync on staging |
[production] |
| 10:42 |
<hnowlan@deploy1002> |
helmfile [eqiad] START helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 10:42 |
<hnowlan@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 10:42 |
<hnowlan@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: sync on staging |
[production] |
| 10:42 |
<hnowlan@deploy1002> |
helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 10:41 |
<hnowlan@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 10:40 |
<hnowlan@deploy1002> |
helmfile [codfw] DONE helmfile.d/services/changeprop-jobqueue: sync on staging |
[production] |
| 10:40 |
<hnowlan@deploy1002> |
helmfile [codfw] START helmfile.d/services/changeprop-jobqueue: sync on production |
[production] |
| 10:39 |
<jmm@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host ganeti1021.eqiad.wmnet with OS buster |
[production] |
| 10:11 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reimage for host ganeti1021.eqiad.wmnet with OS buster |
[production] |
| 10:05 |
<jelto@deploy1002> |
helmfile [staging] DONE helmfile.d/services/termbox: apply |
[production] |
| 10:05 |
<jelto@deploy1002> |
helmfile [staging] START helmfile.d/services/termbox: apply |
[production] |
| 09:29 |
<kevinbazira@deploy1002> |
helmfile [ml-serve-codfw] Ran 'sync' command on namespace 'revscoring-editquality' for release 'main' . |
[production] |
| 09:29 |
<kevinbazira@deploy1002> |
helmfile [ml-serve-eqiad] Ran 'sync' command on namespace 'revscoring-editquality' for release 'main' . |
[production] |
| 09:02 |
<marostegui@cumin1001> |
dbctl commit (dc=all): 'Remove watchlist group from s1 eqiad T263127', diff saved to https://phabricator.wikimedia.org/P20599 and previous config saved to /var/cache/conftool/dbconfig/20220211-090223-marostegui.json |
[production] |
| 08:57 |
<jmm@cumin2002> |
END (ERROR) - Cookbook sre.hosts.reimage (exit_code=97) for host ganeti1011.eqiad.wmnet with OS buster |
[production] |
| 08:36 |
<jmm@cumin2002> |
START - Cookbook sre.hosts.reimage for host ganeti1011.eqiad.wmnet with OS buster |
[production] |
| 06:23 |
<marostegui@cumin1001> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2 days, 0:00:00 on 8 hosts with reason: Maintenance |
[production] |
| 06:23 |
<marostegui@cumin1001> |
START - Cookbook sre.hosts.downtime for 2 days, 0:00:00 on 8 hosts with reason: Maintenance |
[production] |