|
2024-02-28
§
|
| 00:53 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2208.codfw.wmnet with reason: host reimage |
[production] |
| 00:51 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2206.codfw.wmnet with reason: host reimage |
[production] |
| 00:51 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2207.codfw.wmnet with reason: host reimage |
[production] |
| 00:51 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2205.codfw.wmnet with reason: host reimage |
[production] |
| 00:50 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db2203.codfw.wmnet with reason: host reimage |
[production] |
| 00:47 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2203.codfw.wmnet with reason: host reimage |
[production] |
| 00:30 |
<jhancock@cumin2002> |
END (FAIL) - Cookbook sre.hosts.reimage (exit_code=99) for host db2198.codfw.wmnet with OS bookworm |
[production] |
| 00:28 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2208.codfw.wmnet with OS bookworm |
[production] |
| 00:28 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2207.codfw.wmnet with OS bookworm |
[production] |
| 00:28 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2206.codfw.wmnet with OS bookworm |
[production] |
| 00:28 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2205.codfw.wmnet with OS bookworm |
[production] |
| 00:28 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2204.codfw.wmnet with OS bookworm |
[production] |
| 00:28 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2203.codfw.wmnet with OS bookworm |
[production] |
| 00:10 |
<rzl@deploy2002> |
helmfile [codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 00:10 |
<rzl@deploy2002> |
helmfile [codfw] START helmfile.d/admin 'apply'. |
[production] |
| 00:08 |
<rzl@deploy2002> |
helmfile [staging-eqiad] DONE helmfile.d/admin 'apply'. |
[production] |
| 00:08 |
<rzl@deploy2002> |
helmfile [staging-eqiad] START helmfile.d/admin 'apply'. |
[production] |
| 00:08 |
<rzl@deploy2002> |
helmfile [staging-codfw] DONE helmfile.d/admin 'apply'. |
[production] |
| 00:07 |
<rzl@deploy2002> |
helmfile [staging-codfw] START helmfile.d/admin 'apply'. |
[production] |
| 00:06 |
<bking@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 15:00:00 on wdqs1011.eqiad.wmnet with reason: T355617 |
[production] |
| 00:06 |
<bking@cumin2002> |
START - Cookbook sre.hosts.downtime for 15:00:00 on wdqs1011.eqiad.wmnet with reason: T355617 |
[production] |
| 00:02 |
<dzahn@cumin1002> |
END (PASS) - Cookbook sre.ganeti.makevm (exit_code=0) for new host contint1003.eqiad.wmnet |
[production] |
| 00:02 |
<dzahn@cumin1002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host contint1003.eqiad.wmnet with OS bullseye |
[production] |
| 00:02 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db2199.codfw.wmnet with OS bookworm |
[production] |
| 00:01 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002" |
[production] |
| 00:00 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db2202.codfw.wmnet with OS bookworm |
[production] |
| 00:00 |
<jhancock@cumin2002> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002" |
[production] |
|
2024-02-27
§
|
| 23:57 |
<mutante> |
T358237 - manually went through "fix forward"-steps from T349619 (install puppet-agent package, delete old key material, create new CSR, sign on puppetserver, node clean on puppetmaster) to fix puppet failures while makevm cookbook still running (which couldn't find succesful puppet run) |
[production] |
| 23:54 |
<jhancock@cumin2002> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002" |
[production] |
| 23:54 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db2201.codfw.wmnet with OS bookworm |
[production] |
| 23:54 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002" |
[production] |
| 23:52 |
<mutante> |
T358237 - creating VM with cookbook fails because puppet runs have certificate issue, applied role is already migrated to puppet 7 though |
[production] |
| 23:50 |
<jhancock@cumin2002> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002" |
[production] |
| 23:49 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host db2200.codfw.wmnet with OS bookworm |
[production] |
| 23:49 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002" |
[production] |
| 23:45 |
<jhancock@cumin2002> |
START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - jhancock@cumin2002" |
[production] |
| 23:40 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db2199.codfw.wmnet with reason: host reimage |
[production] |
| 23:38 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db2202.codfw.wmnet with reason: host reimage |
[production] |
| 23:36 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db2201.codfw.wmnet with reason: host reimage |
[production] |
| 23:33 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2199.codfw.wmnet with reason: host reimage |
[production] |
| 23:33 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2202.codfw.wmnet with reason: host reimage |
[production] |
| 23:33 |
<jhancock@cumin2002> |
END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on db2200.codfw.wmnet with reason: host reimage |
[production] |
| 23:33 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2201.codfw.wmnet with reason: host reimage |
[production] |
| 23:30 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.downtime for 2:00:00 on db2200.codfw.wmnet with reason: host reimage |
[production] |
| 23:10 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2202.codfw.wmnet with OS bookworm |
[production] |
| 23:10 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2201.codfw.wmnet with OS bookworm |
[production] |
| 23:10 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2200.codfw.wmnet with OS bookworm |
[production] |
| 23:10 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2199.codfw.wmnet with OS bookworm |
[production] |
| 23:10 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2198.codfw.wmnet with OS bookworm |
[production] |
| 23:09 |
<jhancock@cumin2002> |
START - Cookbook sre.hosts.reimage for host db2197.codfw.wmnet with OS bookworm |
[production] |