production SAL

2951-3000 of 10000 results (84ms)

2023-08-14 §
10:38	<stevemunene@cumin1001>	START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.ganeti.makevm: created new VM an-airflow1007.eqiad.wmnet - stevemunene@cumin1001"	[production]
10:38	<cmooney@cumin1001>	START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Rename mr1-esams-new to mr1-esams in dns. - cmooney@cumin1001"	[production]
10:38	<stevemunene@cumin1001>	END (PASS) - Cookbook sre.dns.wipe-cache (exit_code=0) an-airflow1007.eqiad.wmnet on all recursors	[production]
10:38	<stevemunene@cumin1001>	START - Cookbook sre.dns.wipe-cache an-airflow1007.eqiad.wmnet on all recursors	[production]
10:36	<cmooney@cumin1001>	START - Cookbook sre.dns.netbox	[production]
10:34	<cmooney@cumin1001>	END (PASS) - Cookbook sre.dns.netbox (exit_code=0)	[production]
10:34	<cmooney@cumin1001>	END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Rename mr1-esams-new to mr1-esams in dns. - cmooney@cumin1001"	[production]
10:33	<cmooney@cumin1001>	START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Rename mr1-esams-new to mr1-esams in dns. - cmooney@cumin1001"	[production]
10:30	<cmooney@cumin1001>	START - Cookbook sre.dns.netbox	[production]
10:26	<stevemunene@cumin1001>	END (FAIL) - Cookbook sre.dns.netbox (exit_code=99)	[production]
10:25	<cmooney@cumin1001>	END (PASS) - Cookbook sre.dns.netbox (exit_code=0)	[production]
10:25	<cmooney@cumin1001>	END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Rename mr1-esams-new to mr1-esams in dns. - cmooney@cumin1001"	[production]
10:24	<btullis@cumin1001>	END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host stat1008.eqiad.wmnet	[production]
10:24	<cmooney@cumin1001>	START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Rename mr1-esams-new to mr1-esams in dns. - cmooney@cumin1001"	[production]
10:20	<btullis@cumin1001>	END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host stat1009.eqiad.wmnet	[production]
10:13	<stevemunene@cumin1001>	START - Cookbook sre.dns.netbox	[production]
10:13	<stevemunene@cumin1001>	START - Cookbook sre.ganeti.makevm for new host an-airflow1007.eqiad.wmnet	[production]
10:12	<btullis@cumin1001>	START - Cookbook sre.hosts.reboot-single for host stat1009.eqiad.wmnet	[production]
09:59	<btullis@cumin1001>	END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host an-worker1124.eqiad.wmnet	[production]
09:49	<btullis@cumin1001>	START - Cookbook sre.hosts.reboot-single for host stat1008.eqiad.wmnet	[production]
09:48	<btullis@cumin1001>	END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host stat1007.eqiad.wmnet	[production]
09:41	<cmooney@cumin1001>	START - Cookbook sre.dns.netbox	[production]
09:41	<cmooney@cumin1001>	END (FAIL) - Cookbook sre.dns.netbox (exit_code=99)	[production]
09:39	<btullis@cumin1001>	START - Cookbook sre.hosts.reboot-single for host stat1007.eqiad.wmnet	[production]
09:37	<btullis@cumin1001>	END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host stat1006.eqiad.wmnet	[production]
09:32	<cmooney@cumin1001>	START - Cookbook sre.dns.netbox	[production]
09:32	<cmooney@cumin1001>	END (ERROR) - Cookbook sre.dns.netbox (exit_code=97)	[production]
09:32	<cmooney@cumin1001>	START - Cookbook sre.dns.netbox	[production]
09:28	<btullis@cumin1001>	START - Cookbook sre.hosts.reboot-single for host stat1006.eqiad.wmnet	[production]
09:27	<btullis@cumin1001>	END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host stat1005.eqiad.wmnet	[production]
09:26	<btullis@cumin1001>	START - Cookbook sre.hosts.reboot-single for host an-worker1124.eqiad.wmnet	[production]
09:16	<btullis@cumin1001>	START - Cookbook sre.hosts.reboot-single for host stat1005.eqiad.wmnet	[production]
09:13	<btullis@cumin1001>	END (PASS) - Cookbook sre.hosts.reboot-single (exit_code=0) for host stat1004.eqiad.wmnet	[production]
09:11	<cmooney@cumin1001>	END (PASS) - Cookbook sre.dns.netbox (exit_code=0)	[production]
09:11	<cmooney@cumin1001>	END (PASS) - Cookbook sre.puppet.sync-netbox-hiera (exit_code=0) generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Rename mr1-esams dns to mr1-eams-old. - cmooney@cumin1001"	[production]
09:10	<cmooney@cumin1001>	START - Cookbook sre.puppet.sync-netbox-hiera generate netbox hiera data: "Triggered by cookbooks.sre.dns.netbox: Rename mr1-esams dns to mr1-eams-old. - cmooney@cumin1001"	[production]
09:08	<cmooney@cumin1001>	START - Cookbook sre.dns.netbox	[production]
09:02	<btullis@cumin1001>	START - Cookbook sre.hosts.reboot-single for host stat1004.eqiad.wmnet	[production]
2023-08-13 §
16:07	<topranks>	powering down cr3-esams	[production]
16:05	<topranks>	powering down cr2-esams	[production]
15:54	<topranks>	Disabling esams peering at AMS-IX prior to removing router	[production]
15:45	<topranks>	Disable transport cct cr2-esams to cr2-eqiad prior to disconnect T329219	[production]
15:26	<topranks>	disable transit and peering links on cr2-esams & cr3-esams before decom T329219	[production]
2023-08-12 §
08:25	<ladsgroup@cumin1001>	END (PASS) - Cookbook sre.hosts.downtime (exit_code=0) for 1 day, 0:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance	[production]
08:25	<ladsgroup@cumin1001>	START - Cookbook sre.hosts.downtime for 1 day, 0:00:00 on dbstore1003.eqiad.wmnet with reason: Maintenance	[production]
08:25	<ladsgroup@cumin1001>	dbctl commit (dc=all): 'Repooling after maintenance db1219 (T342617)', diff saved to https://phabricator.wikimedia.org/P50569 and previous config saved to /var/cache/conftool/dbconfig/20230812-082511-ladsgroup.json	[production]
08:10	<ladsgroup@cumin1001>	dbctl commit (dc=all): 'Repooling after maintenance db1219', diff saved to https://phabricator.wikimedia.org/P50568 and previous config saved to /var/cache/conftool/dbconfig/20230812-081005-ladsgroup.json	[production]
07:54	<ladsgroup@cumin1001>	dbctl commit (dc=all): 'Repooling after maintenance db1219', diff saved to https://phabricator.wikimedia.org/P50567 and previous config saved to /var/cache/conftool/dbconfig/20230812-075459-ladsgroup.json	[production]
07:39	<ladsgroup@cumin1001>	dbctl commit (dc=all): 'Repooling after maintenance db1219 (T342617)', diff saved to https://phabricator.wikimedia.org/P50566 and previous config saved to /var/cache/conftool/dbconfig/20230812-073953-ladsgroup.json	[production]
05:56	<ladsgroup@cumin1001>	dbctl commit (dc=all): 'Depooling db1219 (T342617)', diff saved to https://phabricator.wikimedia.org/P50565 and previous config saved to /var/cache/conftool/dbconfig/20230812-055651-ladsgroup.json	[production]