| 
      
        2023-01-13
      
      ยง
     | 
  
    
  | 15:19 | 
  <jmm@cumin2002> | 
  START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "new bastion - jmm@cumin2002" | 
  [production] | 
            
  | 15:18 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hosts.reimage (exit_code=0) for host an-coord1003.eqiad.wmnet with OS bullseye | 
  [production] | 
            
  | 15:18 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002" | 
  [production] | 
            
  | 15:17 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-coord1004.eqiad.wmnet with reason: host reimage | 
  [production] | 
            
  | 15:14 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.downtime for 2:00:00 on an-coord1004.eqiad.wmnet with reason: host reimage | 
  [production] | 
            
  | 15:11 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.hosts.reimage: Host reimage - pt1979@cumin2002" | 
  [production] | 
            
  | 15:01 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.reimage for host an-coord1004.eqiad.wmnet with OS bullseye | 
  [production] | 
            
  | 14:57 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 2:00:00 on an-coord1003.eqiad.wmnet with reason: host reimage | 
  [production] | 
            
  | 14:54 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.downtime for 2:00:00 on an-coord1003.eqiad.wmnet with reason: host reimage | 
  [production] | 
            
  | 14:49 | 
  <volans> | 
  uploaded cumin_4.2.0 to apt.wikimedia.org bullseye-wikimedia | 
  [production] | 
            
  | 14:34 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.reimage for host an-coord1003.eqiad.wmnet with OS bullseye | 
  [production] | 
            
  | 12:48 | 
  <moritzm> | 
  installing bast6002 T324974 | 
  [production] | 
            
  | 12:39 | 
  <jelto@cumin1001> | 
  END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1:00:00 on gitlab2002.wikimedia.org with reason: troubeleshoot backup restore on gitlab replica | 
  [production] | 
            
  | 12:38 | 
  <jelto@cumin1001> | 
  START - Cookbook sre.hosts.downtime for 1:00:00 on gitlab2002.wikimedia.org with reason: troubeleshoot backup restore on gitlab replica | 
  [production] | 
            
  | 11:48 | 
  <jmm@cumin2002> | 
  END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "new bastions - jmm@cumin2002" | 
  [production] | 
            
  | 11:45 | 
  <jmm@cumin2002> | 
  START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "new bastions - jmm@cumin2002" | 
  [production] | 
            
  | 10:53 | 
  <moritzm> | 
  installing bast5003 T324974 | 
  [production] | 
            
  | 10:49 | 
  <jynus> | 
  restarting update-ubuntu-mirror on mirror1001 due to remote server concurrency issues | 
  [production] | 
            
  | 09:41 | 
  <moritzm> | 
  installing bast4004 T324974 | 
  [production] | 
            
  | 09:06 | 
  <moritzm> | 
  installing bast3006 T324974 | 
  [production] | 
            
  | 02:33 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host druid1011.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 02:09 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.provision for host druid1011.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 02:08 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host druid1010.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 02:03 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host druid1009.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 01:55 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.provision for host druid1009.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 01:54 | 
  <pt1979@cumin2002> | 
  END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host druid1009.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 01:53 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.provision for host druid1009.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 01:52 | 
  <pt1979@cumin2002> | 
  END (FAIL) - Cookbook sre.hosts.provision (exit_code=99) for host druid1009.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 01:36 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.provision for host druid1010.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 01:36 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.provision for host druid1009.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 01:26 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=False) upgrade firmware for hosts ['an-mariadb1002'] | 
  [production] | 
            
  | 01:26 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-mariadb1002'] | 
  [production] | 
            
  | 01:25 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=False) upgrade firmware for hosts ['an-mariadb1001'] | 
  [production] | 
            
  | 01:25 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-mariadb1001'] | 
  [production] | 
            
  | 01:21 | 
  <pt1979@cumin2002> | 
  END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=True) upgrade firmware for hosts ['an-mariadb1002'] | 
  [production] | 
            
  | 01:21 | 
  <pt1979@cumin2002> | 
  END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=True) upgrade firmware for hosts ['an-mariadb1001'] | 
  [production] | 
            
  | 01:04 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-mariadb1002'] | 
  [production] | 
            
  | 01:04 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-mariadb1001'] | 
  [production] | 
            
  | 01:03 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=False) upgrade firmware for hosts ['an-coord1004'] | 
  [production] | 
            
  | 01:03 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-coord1004'] | 
  [production] | 
            
  | 01:02 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hardware.upgrade-firmware (exit_code=False) upgrade firmware for hosts ['an-coord1003'] | 
  [production] | 
            
  | 01:02 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-coord1003'] | 
  [production] | 
            
  | 00:54 | 
  <pt1979@cumin2002> | 
  END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=True) upgrade firmware for hosts ['an-coord1004'] | 
  [production] | 
            
  | 00:54 | 
  <pt1979@cumin2002> | 
  END (FAIL) - Cookbook sre.hardware.upgrade-firmware (exit_code=True) upgrade firmware for hosts ['an-coord1003'] | 
  [production] | 
            
  | 00:41 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-coord1004'] | 
  [production] | 
            
  | 00:40 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hardware.upgrade-firmware upgrade firmware for hosts ['an-coord1003'] | 
  [production] | 
            
  | 00:36 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host an-mariadb1002.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 00:36 | 
  <pt1979@cumin2002> | 
  END (PASS) - Cookbook sre.hosts.provision (exit_code=0) for host an-mariadb1001.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 00:16 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.provision for host an-mariadb1002.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] | 
            
  | 00:15 | 
  <pt1979@cumin2002> | 
  START - Cookbook sre.hosts.provision for host an-mariadb1001.mgmt.eqiad.wmnet with reboot policy FORCED | 
  [production] |