OceanBase集群存在不工作 OBServer

撑伞接落花 · 2025 年7 月 24 日 09:43

【使用环境】生产环境
【 OB or 其他组件】
【使用版本】
【问题描述】集群：CATL_CE_O_ND02，告警：OceanBase集群存在不工作 OBServer。不工作 OBServer 数量为 1，不工作 OBServer 有 172.26.104.123。
产生时间 2025年7月24日 08:02:01

select * from dba_ob_servers

发现集群在 08:02:01 左右有大量的rootservice 重启操作，
select * from DBA_OB_ROOTSERVICE_EVENT_HISTORY order by 1 desc limit 600

32:18.9	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083A4ED7966-0-0	task_id	1.75332E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
32:16.1	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687C-000630CF32BC5F1A-0-0	task_id	1.75332E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
31:03.2	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063084C49BD4F9-0-0	task_id	1.75332E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
31:00.4	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063084C49BD4DB-0-0	task_id	1.75332E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
25:54.8	root_service	admin_refresh_io_calibration	ret	0												172.26.104.123	2882
25:50.7	root_service	finish_wait_stop	cost	2503												172.26.104.124	2882
25:50.7	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.124:2882								172.26.104.124	2882
25:49.8	root_service	full_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
25:49.8	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
25:49.8	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
25:49.8	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
25:49.8	root_service	start_rootservice	self_addr	172.26.104.123:2882												172.26.104.123	2882
25:49.8	root_service	finish_wait_stop	cost	142459												172.26.104.123	2882
25:49.6	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.123:2882								172.26.104.123	2882
25:46.4	root_service	stop_rootservice	self_addr	172.26.104.124:2882												172.26.104.124	2882
25:44.9	root_service	full_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
25:44.8	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
25:44.8	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
25:44.5	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
25:44.5	root_service	start_rootservice	self_addr	172.26.104.124:2882												172.26.104.124	2882
25:43.0	root_service	stop_rootservice	self_addr	172.26.104.123:2882												172.26.104.123	2882
19:47.8	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063084B30BFA40-0-0	task_id	1.75332E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
19:44.9	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063084B30BFA20-0-0	task_id	1.75332E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
05:33.2	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083A67D41C7-0-0	task_id	1.75332E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
05:30.4	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083A67D41BA-0-0	task_id	1.75332E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
28:23.3	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083ED4C5F74-0-0	task_id	1.75332E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
28:20.4	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687C-000630CED0ACA17A-0-0	task_id	1.75332E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
28:17.3	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083988D2780-0-0	task_id	1.75332E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
28:14.4	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083988D276C-0-0	task_id	1.75332E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
17:04.2	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-000630841F9C24F7-0-0	task_id	1.75332E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
17:01.4	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-000630841F9C24E6-0-0	task_id	1.75332E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
04:16.5	root_service	admin_refresh_io_calibration	ret	0												172.26.104.123	2882
04:11.5	root_service	full_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
04:11.5	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
04:11.5	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
04:11.5	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
04:11.5	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
04:11.5	root_service	start_rootservice	self_addr	172.26.104.123:2882												172.26.104.123	2882
04:11.5	root_service	finish_wait_stop	cost	138713												172.26.104.123	2882
04:11.3	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.123:2882								172.26.104.123	2882
03:58.9	server	rootserver	server	172.26.104.123:2882												172.26.104.123	2882
03:58.5	root_service	finish_wait_stop	cost	167461												172.26.104.124	2882
03:58.3	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.124:2882								172.26.104.124	2882
03:56.3	root_service	stop_rootservice	self_addr	172.26.104.124:2882												172.26.104.124	2882
03:56.0	server	lease_expire	server	172.26.104.125:2882												172.26.104.123	2882
03:56.0	server	lease_expire	server	172.26.104.124:2882												172.26.104.123	2882
03:55.9	root_service	admin_set_config	ret	0	arg	{name:"rootservice_list", value:"172.26.104.124:2882:2881;172.26.104.125:2882:2881;172.26.104.123:2882:2881", comment:"", zone:"", server:"0.0.0.0:0", tenant_name:"", exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}	is_inner	0								172.26.104.124	2882
03:55.9	server	online	server	172.26.104.123:2882												172.26.104.124	2882
03:55.9	server	start_service	server	172.26.104.123:2882												172.26.104.124	2882
03:55.9	server	last_offline_time reset	server	172.26.104.123:2882												172.26.104.124	2882
03:55.9	server	lease_expire	server	172.26.104.123:2882												172.26.104.123	2882
02:53.4	root_service	update_primary_ip_list	tenant_id	1102	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:53.2	root_service	update_primary_ip_list	tenant_id	1104	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:51.5	root_service	update_primary_ip_list	tenant_id	1090	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:51.4	root_service	update_primary_ip_list	tenant_id	1068	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:51.4	root_service	update_primary_ip_list	tenant_id	1062	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:51.3	root_service	update_primary_ip_list	tenant_id	1036	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:51.1	root_service	update_primary_ip_list	tenant_id	1006	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:51.1	root_service	update_primary_ip_list	tenant_id	1030	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:50.9	root_service	update_primary_ip_list	tenant_id	1012	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:50.3	root_service	update_primary_ip_list	tenant_id	1020	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:50.1	root_service	update_primary_ip_list	tenant_id	1048	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:50.1	root_service	update_primary_ip_list	tenant_id	1072	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.9	root_service	update_primary_ip_list	tenant_id	1004	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.8	root_service	update_primary_ip_list	tenant_id	1010	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.8	root_service	update_primary_ip_list	tenant_id	1084	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.3	root_service	update_primary_ip_list	tenant_id	1054	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.3	root_service	update_primary_ip_list	tenant_id	1018	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.3	root_service	update_primary_ip_list	tenant_id	1008	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.3	root_service	update_primary_ip_list	tenant_id	1080	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.3	root_service	update_primary_ip_list	tenant_id	1046	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.3	root_service	update_primary_ip_list	tenant_id	1082	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.2	root_service	update_primary_ip_list	tenant_id	1078	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.2	root_service	update_primary_ip_list	tenant_id	1094	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.2	root_service	update_primary_ip_list	tenant_id	1028	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.2	root_service	update_primary_ip_list	tenant_id	1066	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1056	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1022	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1088	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1100	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1070	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1038	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1074	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1014	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1086	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.1	root_service	update_primary_ip_list	tenant_id	1024	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
02:49.0	root_service	update_primary_ip_list	tenant_id	1034	ret	-4012	ob_error_name	OB_TIMEOUT	ob_error_str	Timeout	primary_user_tenant		primary_ip_list	[]		172.26.104.123	2882
01:48.0	root_service	admin_refresh_io_calibration	ret	0												172.26.104.124	2882
01:47.0	root_service	admin_refresh_io_calibration	ret	-4025												172.26.104.124	2882
01:43.6	server	last_offline_time set	server	172.26.104.123:2882												172.26.104.124	2882
01:43.6	server	lease_expire	server	172.26.104.123:2882												172.26.104.124	2882
01:37.0	root_service	admin_refresh_io_calibration	ret	-4025												172.26.104.124	2882
01:35.6	server	rootserver	server	172.26.104.124:2882												172.26.104.124	2882
01:29.0	server	lease_expire	server	172.26.104.125:2882												172.26.104.124	2882
01:29.0	server	lease_expire	server	172.26.104.124:2882												172.26.104.124	2882
01:29.0	server	lease_expire	server	172.26.104.123:2882												172.26.104.124	2882
01:27.0	root_service	admin_refresh_io_calibration	ret	-4025												172.26.104.124	2882
01:17.0	root_service	admin_refresh_io_calibration	ret	-4025												172.26.104.124	2882
01:06.1	root_service	admin_set_config	ret	0	arg	{name:"rootservice_list", value:"172.26.104.124:2882:2881;172.26.104.125:2882:2881", comment:"", zone:"", server:"0.0.0.0:0", tenant_name:"", exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}	is_inner	0								172.26.104.124	2882
01:03.0	root_service	full_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
01:03.0	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
01:01.2	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
01:01.0	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
01:01.0	root_service	start_rootservice	self_addr	172.26.104.124:2882												172.26.104.124	2882
59:32.4	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-0006308391FCE352-0-0	task_id	1.75332E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
59:29.1	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-0006308391FCE321-0-0	task_id	1.75332E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
37:34.1	root_service	admin_refresh_io_calibration	ret	0												172.26.104.123	2882
37:32.0	root_service	finish_wait_stop	cost	162616												172.26.104.124	2882
37:31.9	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.124:2882								172.26.104.124	2882
37:29.1	root_service	full_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
37:29.1	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
37:27.8	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
37:27.8	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
37:27.8	root_service	start_rootservice	self_addr	172.26.104.123:2882												172.26.104.123	2882
37:27.3	root_service	stop_rootservice	self_addr	172.26.104.124:2882												172.26.104.124	2882
37:26.5	root_service	admin_set_config	ret	0	arg	{name:"rootservice_list", value:"172.26.104.124:2882:2881;172.26.104.125:2882:2881;172.26.104.123:2882:2881", comment:"", zone:"", server:"0.0.0.0:0", tenant_name:"", exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}	is_inner	0								172.26.104.124	2882
37:26.1	root_service	finish_wait_stop	cost	8972369												172.26.104.123	2882
37:26.0	server	lease_expire	server	172.26.104.125:2882												172.26.104.123	2882
37:25.5	root_service	admin_refresh_io_calibration	ret	0												172.26.104.124	2882
37:17.1	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.123:2882								172.26.104.123	2882
37:14.6	root_service	admin_set_config	ret	0	arg	{name:"rootservice_list", value:"172.26.104.124:2882:2881;172.26.104.125:2882:2881", comment:"", zone:"", server:"0.0.0.0:0", tenant_name:"", exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}	is_inner	0								172.26.104.124	2882
37:11.6	root_service	full_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
37:11.6	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
37:09.7	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
37:09.6	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
37:09.6	root_service	start_rootservice	self_addr	172.26.104.124:2882												172.26.104.124	2882
37:09.0	root_service	stop_rootservice	self_addr	172.26.104.123:2882												172.26.104.123	2882
25:39.7	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083A4ECAAB7-0-0	task_id	1.75331E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
25:36.7	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083A4ECAAA1-0-0	task_id	1.75331E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
24:23.5	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083996CA095-0-0	task_id	1.75331E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
24:20.6	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063083996CA06E-0-0	task_id	1.75331E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
20:06.1	root_service	admin_refresh_io_calibration	ret	0												172.26.104.123	2882
20:02.5	root_service	finish_wait_stop	cost	37444												172.26.104.124	2882
20:02.4	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.124:2882								172.26.104.124	2882
20:01.2	root_service	full_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
20:01.1	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
20:01.1	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
20:01.1	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
20:01.1	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
20:01.1	root_service	start_rootservice	self_addr	172.26.104.123:2882												172.26.104.123	2882
20:01.1	root_service	finish_wait_stop	cost	174399												172.26.104.123	2882
20:00.9	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.123:2882								172.26.104.123	2882
20:00.2	server	rootserver	server	172.26.104.123:2882												172.26.104.123	2882
19:57.5	root_service	root_minor_freeze	ret	0	arg	{tenant_ids:[1071], server_list:[], zone:"", tablet_id:{id:0}, ls_id:{id:-1}}										172.26.104.124	2882
19:57.4	root_service	stop_rootservice	self_addr	172.26.104.124:2882												172.26.104.124	2882
19:57.4	root_service	root_minor_freeze	ret	0	arg	{tenant_ids:[1020], server_list:[], zone:"", tablet_id:{id:0}, ls_id:{id:-1}}										172.26.104.124	2882
19:57.4	root_service	root_minor_freeze	ret	0	arg	{tenant_ids:[1039], server_list:[], zone:"", tablet_id:{id:0}, ls_id:{id:-1}}										172.26.104.124	2882
19:57.3	root_service	admin_refresh_io_calibration	ret	0												172.26.104.124	2882
19:57.1	server	lease_expire	server	172.26.104.125:2882												172.26.104.123	2882
19:57.0	server	lease_expire	server	172.26.104.124:2882												172.26.104.123	2882
19:57.0	server	lease_expire	server	172.26.104.123:2882												172.26.104.123	2882
19:56.9	root_service	admin_set_config	ret	0	arg	{name:"rootservice_list", value:"172.26.104.124:2882:2881;172.26.104.125:2882:2881;172.26.104.123:2882:2881", comment:"", zone:"", server:"0.0.0.0:0", tenant_name:"", exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}	is_inner	0								172.26.104.124	2882
19:56.4	server	rootserver	server	172.26.104.124:2882												172.26.104.124	2882
19:55.7	root_service	root_minor_freeze	ret	0	arg	{tenant_ids:[1061], server_list:[], zone:"", tablet_id:{id:0}, ls_id:{id:-1}}										172.26.104.124	2882
19:52.4	server	lease_expire	server	172.26.104.125:2882												172.26.104.124	2882
19:52.4	server	lease_expire	server	172.26.104.124:2882												172.26.104.124	2882
19:52.4	server	lease_expire	server	172.26.104.123:2882												172.26.104.124	2882
19:50.5	root_service	admin_refresh_io_calibration	ret	-4025												172.26.104.124	2882
19:40.5	root_service	admin_refresh_io_calibration	ret	-4025												172.26.104.124	2882
19:29.5	root_service	admin_set_config	ret	0	arg	{name:"rootservice_list", value:"172.26.104.124:2882:2881;172.26.104.125:2882:2881", comment:"", zone:"", server:"0.0.0.0:0", tenant_name:"", exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}	is_inner	0								172.26.104.124	2882
19:26.4	root_service	full_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
19:26.4	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
19:26.4	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
19:24.4	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
14:19.6	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-000630841F9BAAAA-0-0	task_id	1.75331E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
14:16.7	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-000630841F9BAA8F-0-0	task_id	1.75331E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
56:49.3	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-0006308391FC8B58-0-0	task_id	1.75331E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
56:46.4	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687B-000630D8840D6CFF-0-0	task_id	1.75331E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882
53:35.6	root_service	admin_refresh_io_calibration	ret	0												172.26.104.123	2882
53:31.0	root_service	finish_wait_stop	cost	69166												172.26.104.124	2882
53:31.0	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.124:2882								172.26.104.124	2882
53:30.6	root_service	full_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
53:30.6	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
53:30.6	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
53:30.6	server	load_servers	ret	0	has_build	1										172.26.104.123	2882
53:30.6	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.123:2882										172.26.104.123	2882
53:30.6	root_service	start_rootservice	self_addr	172.26.104.123:2882												172.26.104.123	2882
53:30.5	root_service	finish_wait_stop	cost	187630												172.26.104.123	2882
53:30.4	root_service	finish_stop_thread	ret	0	ret	OB_SUCCESS	self_addr	172.26.104.123:2882								172.26.104.123	2882
53:27.5	server	rootserver	server	172.26.104.123:2882												172.26.104.123	2882
53:24.9	root_service	stop_rootservice	self_addr	172.26.104.124:2882												172.26.104.124	2882
53:21.6	server	rootserver	server	172.26.104.124:2882												172.26.104.124	2882
53:20.1	root_service	admin_set_config	ret	0	arg	{name:"rootservice_list", value:"172.26.104.124:2882:2881;172.26.104.125:2882:2881;172.26.104.123:2882:2881", comment:"", zone:"", server:"0.0.0.0:0", tenant_name:"", exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}	is_inner	0								172.26.104.124	2882
53:19.9	server	lease_expire	server	172.26.104.125:2882												172.26.104.123	2882
53:19.8	root_service	admin_refresh_io_calibration	ret	0												172.26.104.124	2882
53:19.5	root_service	root_minor_freeze	ret	0	arg	{tenant_ids:[1052], server_list:[], zone:"", tablet_id:{id:0}, ls_id:{id:-1}}										172.26.104.124	2882
53:19.4	root_service	root_minor_freeze	ret	0	arg	{tenant_ids:[1079], server_list:[], zone:"", tablet_id:{id:0}, ls_id:{id:-1}}										172.26.104.124	2882
53:14.6	root_service	admin_refresh_io_calibration	ret	-4025												172.26.104.124	2882
53:00.6	root_service	admin_set_config	ret	0	arg	{name:"rootservice_list", value:"172.26.104.124:2882:2881;172.26.104.125:2882:2881", comment:"", zone:"", server:"0.0.0.0:0", tenant_name:"", exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}	is_inner	0								172.26.104.124	2882
53:00.6	root_service	admin_set_config	ret	0	arg	{name:"rootservice_list", value:"172.26.104.124:2882:2881;172.26.104.125:2882:2881", comment:"", zone:"", server:"0.0.0.0:0", tenant_name:"", exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}	is_inner	0								172.26.104.124	2882
53:00.6	root_service	full_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
53:00.6	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
53:00.6	server	load_servers	ret	0	has_build	1										172.26.104.124	2882
52:58.6	root_service	finish_start_rootservice	result	0	self_addr	172.26.104.124:2882										172.26.104.124	2882
52:58.6	root_service	start_rootservice	self_addr	172.26.104.124:2882												172.26.104.124	2882
52:56.0	root_service	stop_rootservice	self_addr	172.26.104.123:2882												172.26.104.123	2882
21:47.1	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063082AF6FECC7-0-0	task_id	1.75331E+15	table_name	dam_domain_department_count	schema_version	515884	{val:1753293604072241000, v:0}	172.26.104.123	2882
21:44.2	ddl scheduler	truncate table new	tenant_id	1044	ret	0	trace_id	YB42AC1A687D-00063082AF6FEC9E-0-0	task_id	1.75331E+15	table_name	dam_domain_detail_count	schema_version	521018	{val:1753293604072241000, v:0}	172.26.104.123	2882

淇铭 · 2025 年7 月 24 日 10:03

在ocp上查看一下集群信息 ob的版本 ocp的版本在提供一下在这个时间三个节点的rootservice.log和observer.log日志

撑伞接落花 · 2025 年7 月 24 日 10:04

ob版本：4.3.5.0

撑伞接落花 · 2025 年7 月 24 日 10:05

ocp：4.3.1-20240805192406

淇铭 · 2025 年7 月 24 日 10:11

这个信息DBA_OB_ROOTSERVICE_EVENT_HISTORY查询能放到txt的文档里么上面的信息太乱了

撑伞接落花 · 2025 年7 月 24 日 10:53

有邮箱没，提供一个

旭辉 · 2025 年7 月 24 日 13:39

可以使用obdiag分析下故障时间点前后5分钟左右的日志

obdiag analyze log [options]

https://www.oceanbase.com/docs/common-obdiag-cn-1000000003242094

淇铭 · 2025 年7 月 24 日 14:00

这个时间的日志有么？目前你们发的日志都是故障以后的日志信息或者按照楼上使用obdiag分析一下日志
目前从DBA_OB_ROOTSERVICE_EVENT_HISTORY视图还有后来发的日志来看有io超时4012的错误

撑伞接落花 · 2025 年7 月 24 日 15:08

[root@obs24 appadmin]#  cat /home/appadmin/obdiag_analyze_pack_20250724145803/result_details.txt 

Analyze OceanBase Online Log Summary:
+----------------+-----------+----------------------------------------------------------------------------------------+----------------------------+-------------+-------------------+--------
-+
| Node           | Status    | FileName                                                                               | First Found Time           |   ErrorCode | Message           |   Count
 |
+================+===========+========================================================================================+============================+=============+===================+========
=+
| 172.26.104.123 | Completed | /home/appadmin/obdiag_analyze_pack_20250724145803/local/observer.log.20250724080337735 | 2025-07-24 08:02:51.621270 |       -4024 | Buffer not enough |    1073
 |
+----------------+-----------+----------------------------------------------------------------------------------------+----------------------------+-------------+-------------------+--------
-+
| 172.26.104.123 | Completed | /home/appadmin/obdiag_analyze_pack_20250724145803/local/observer.log.20250724080425499 | 2025-07-24 08:03:37.779600 |       -4024 | Buffer not enough |    1348
 |
+----------------+-----------+----------------------------------------------------------------------------------------+----------------------------+-------------+-------------------+--------
-+
| 172.26.104.124 | PASS      |                                                                                        |                            |             |                   |        
 |
+----------------+-----------+----------------------------------------------------------------------------------------+----------------------------+-------------+-------------------+--------
-+
| 172.26.104.125 | PASS      |                                                                                        |                            |             |                   |        
 |
+----------------+-----------+----------------------------------------------------------------------------------------+----------------------------+-------------+-------------------+--------
-+

Details:

Node: 172.26.104.123
Status: Completed
FileName: /home/appadmin/obdiag_analyze_pack_20250724145803/local/observer.log.20250724080337735
First Found Time: 2025-07-24 08:02:51.621270
ErrorCode: -4024
Message: Buffer not enough
Count: 1073
Last Found Time: 2025-07-24 08:03:37.691816
Cause: Internal Error
Solution: Contact OceanBase Support
Trace_IDS: ['0-0000000000000000-0-0']

Node: 172.26.104.123
Status: Completed
FileName: /home/appadmin/obdiag_analyze_pack_20250724145803/local/observer.log.20250724080425499
First Found Time: 2025-07-24 08:03:37.779600
ErrorCode: -4024
Message: Buffer not enough
Count: 1348
Last Found Time: 2025-07-24 08:03:56.935425
Cause: Internal Error
Solution: Contact OceanBase Support
Trace_IDS: ['0-0000000000000000-0-0', 'B42AC1A687C-0006307EB1AB7242-0-0', 'B42AC1A687C-000630CBCACB715C-0-0', 'B42AC1A687C-0006307F9F1B73A4-0-0', 'B42AC1A687C-00063028576B73E5-0-0', 'B42AC1A
687C-000630A1DB6C1C3D-0-0', 'B42AC1A687C-00063088FFCB726D-0-0', 'B42AC1A687C-0006302F580B7396-0-0', 'B42AC1A687C-00063020BFAB73C4-0-0', 'B42AC1A687C-000630383BAB722B-0-0', 'B42AC1A687C-00063
028F47B73B6-0-0', 'B42AC1A687C-00063021C65B732C-0-0', 'B42AC1A687C-0006307EB73B7264-0-0', 'B42AC1A687C-00063067C39B731D-0-0', 'B42AC1A687C-0006306ECD9B7324-0-0', 'B42AC1A687C-000630878C8B73F
5-0-0', 'B42AC1A687C-000630383DBB721A-0-0', 'B42AC1A687C-0006307EA23B721D-0-0', 'B42AC1A687C-0006303851AB7221-0-0', 'B42AC1A687C-00063028F57B73C1-0-0', 'B42AC1A687C-00063027758B72FA-0-0', 'B
42AC1A687C-0006309FFAEB732B-0-0', 'B42AC1A687C-00063088B73B7278-0-0', 'B42AC1A687C-0006307EA3BB7220-0-0', 'B42AC1A687C-0006301DD74B7329-0-0', 'B42AC1A687C-0006307C5EAB7411-0-0', 'B42AC1A687C
-0006303844AB722A-0-0', 'B42AC1A687C-00063036000B7327-0-0', 'B42AC1A687C-0006306EC7FB732F-0-0', 'B42AC1A687C-000630383C9B7229-0-0', 'B42AC1A687C-000630B8F6BD5EBC-0-0', 'B42AC1A687C-000630384
B1B7217-0-0', 'B42AC1A687C-00063028F47B73B9-0-0', 'B42AC1A687C-0006301DDD7B73D5-0-0', 'B42AC1A687C-00063027747B7344-0-0', 'B42AC1A687C-0006306534AB73C3-0-0', 'B42AC1A687C-00063020F23B7330-0-
0', 'B42AC1A687C-00063038434B722F-0-0', 'B42AC1A687C-00063022F02B73C9-0-0', 'B42AC1A687C-00063038384B723B-0-0', 'B42AC1A687C-0006307FA44B73C5-0-0', 'B42AC1A687C-00063022A31B7396-0-0', 'B42AC
1A687C-0006303561AB731B-0-0', 'B42AC1A687C-0006307E970B7231-0-0', 'B42AC1A687C-00063065363B7344-0-0', 'B42AC1A687C-0006301DD74B7328-0-0', 'B42AC1A687C-0006305B0D4B73D2-0-0', 'B42AC1A687C-000
630383A0B7218-0-0', 'B42AC1A687C-00063023064B7377-0-0', 'B42AC1A687C-00063038391B720E-0-0', 'B42AC1A687C-00063067B73B73D4-0-0', 'B42AC1A687C-000630A367DB730B-0-0', 'B42AC1A687C-0006309FFAEB7
32C-0-0', 'B42AC1A687C-0006307FA44B73C4-0-0', 'B42AC1A687C-00063038400B7235-0-0', 'B42AC1A687C-000630A367DB730A-0-0', 'B42AC1A687C-000630B328CE755A-0-0', 'B42AC1A687C-0006307EABAB7227-0-0', 
'B42AC1A687C-000630CBCACB715B-0-0', 'B42AC1A687C-0006302038FB73C2-0-0', 'B42AC1A687C-0006305B0D4B73D5-0-0', 'B42AC1A687C-0006301DD18B7399-0-0', 'B42AC1A687C-0006306EC7FB7330-0-0', 'B42AC1A68
7C-0006302038FB73C1-0-0', 'B42AC1A687C-00063020F23B732F-0-0', 'B42AC1A687C-00063068401B732A-0-0', 'B42AC1A687C-00063027758B72F9-0-0', 'B42AC1A687C-0006307EA04B7224-0-0', 'B42AC1A687C-0006307
EAD7B723E-0-0', 'B42AC1A687C-0006303832AB7233-0-0', 'B42AC1A687C-000630383F1B7222-0-0']

Node: 172.26.104.124
Status: PASS
FileName: None
First Found Time: None
ErrorCode: None
Message: None
Count: None
Last Found Time: None
Cause: None
Solution: None
Trace_IDS: None

Node: 172.26.104.125
Status: PASS
FileName: None
First Found Time: None
ErrorCode: None
Message: None
Count: None
Last Found Time: None
Cause: None
Solution: None
Trace_IDS: None

淇铭 · 2025 年7 月 24 日 15:14

你把obdiag巡检的结果发一下吧

撑伞接落花 · 2025 年7 月 24 日 15:25

这个是巡检 123 节点的结果

[root@obs24 appadmin]#  obdiag analyze log  --from "2025-07-24 08:00:00" --to "2025-07-24 08:10:00" \
>   --config obcluster.servers.nodes[0].ip=172.26.104.123 \
>   --config obcluster.servers.global.ssh_username=admin \
>   --config obcluster.servers.global.ssh_password=XXXXXX\
>   --config obcluster.servers.global.home_path=/home/admin/oceanbase
obdiag version: 3.5.0
analyze_log start ...
analyze nodes's log start. Please wait a moment...
analyze start ok

FileListInfo:
+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------+
| Node           | LogList                                                                                                                                                                    
                                                                                                                                                                                              
                                                                                                                                                                                              
                                                                                                                                                                                              
                                                                                                                                                                                              
                                                                            |
+================+============================================================================================================================================================================
==============================================================================================================================================================================================
==============================================================================================================================================================================================
==============================================================================================================================================================================================
==============================================================================================================================================================================================
============================================================================+
| 172.26.104.123 | ['election.log.20250724084258479', 'election.log.wf.20250724084258479', 'observer.log.20250724080018967', 'observer.log.20250724080154954', 'observer.log.20250724080250532
', 'observer.log.20250724080337735', 'observer.log.20250724080425499', 'observer.log.20250724080528832', 'observer.log.20250724080634212', 'observer.log.20250724080734360', 'observer.log.202
50724080839901', 'observer.log.20250724080947678', 'observer.log.20250724081054718', 'observer.log.wf.20250724080018967', 'observer.log.wf.20250724080154954', 'observer.log.wf.20250724080250
532', 'observer.log.wf.20250724080337735', 'observer.log.wf.20250724080425499', 'observer.log.wf.20250724080528832', 'observer.log.wf.20250724080634212', 'observer.log.wf.20250724080734360',
 'observer.log.wf.20250724080839901', 'observer.log.wf.20250724080947678', 'observer.log.wf.20250724081054718', 'rootservice.log.20250724080526985', 'rootservice.log.20250724081916395', 'roo
tservice.log.wf.20250724080526985', 'rootservice.log.wf.20250724081916395'] |
+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------------------------------------------+


Analyze OceanBase Online Log Summary:

+----------------+-----------+----------------------------------------------------------------------------------------+----------------------------+-------------+-------------------+--------
-+
| Node           | Status    | FileName                                                                               | First Found Time           |   ErrorCode | Message           |   Count
 |
+================+===========+========================================================================================+============================+=============+===================+========
=+
| 172.26.104.123 | Completed | /home/appadmin/obdiag_analyze_pack_20250724152253/local/observer.log.20250724080337735 | 2025-07-24 08:02:51.621270 |       -4024 | Buffer not enough |    1073
 |
+----------------+-----------+----------------------------------------------------------------------------------------+----------------------------+-------------+-------------------+--------
-+
| 172.26.104.123 | Completed | /home/appadmin/obdiag_analyze_pack_20250724152253/local/observer.log.20250724080425499 | 2025-07-24 08:03:37.779600 |       -4024 | Buffer not enough |    1348
 |
+----------------+-----------+----------------------------------------------------------------------------------------+----------------------------+-------------+-------------------+--------
-+
For more details, please run cmd ' cat /home/appadmin/obdiag_analyze_pack_20250724152253/result_details.txt '

Trace ID: 0152058e-685f-11f0-a24f-00109b5d7d08
If you want to view detailed obdiag logs, please run: obdiag display-trace 0152058e-685f-11f0-a24f-00109b5d7d08

淇铭 · 2025 年7 月 24 日 15:40

你把obdiag巡检以后生成的有个压缩文件你把压缩文件发一下建议你还是放到你们外发的地址上我去下载

淇铭 · 2025 年7 月 25 日 09:54

检查一下 ob集群的时钟是否一致把obdiag分析的结果压缩文件发一下再看看问题

sonomx · 2025 年7 月 25 日 10:33

dmesg看下，盘坏了吧

撑伞接落花 · 2025 年7 月 25 日 11:10

已收集日志到企业网盘了，发送连接到您邮件

淇铭 · 2025 年7 月 25 日 11:29

地址应该有问题打不开

淇铭 · 2025 年7 月 25 日 14:56

当时有什么操作么？例如查询语句慢或者其他的表现还是突然查看ocp告警有个节点不工作了

淇铭 · 2025 年7 月 25 日 15:59

这两个信息查一下看看
SELECT * FROM oceanbase.__all_server_event_history WHERE module=“election” AND value1=1 AND value2=1 ORDER BY gmt_create desc limit 20;
SELECT * FROM __all_server_event_history WHERE module=“log” AND value1=1 AND value2=1 AND event like “%ROLE TRANSITION%” ORDER BY gmt_create desc limit 20;

撑伞接落花 · 2025 年7 月 25 日 17:31

邮件重新发你一个地址，看看能打开不？

撑伞接落花 · 2025 年7 月 25 日 17:34

SELECT * FROM oceanbase.__all_server_event_history WHERE module=‘election’ AND value1=1 AND value2=1 ORDER BY gmt_create desc limit 20;

2025-07-25 16:32:22.914966	172.26.104.124	2882	ELECTION	lease expired	TENANT_ID	1	LS_ID	1	INFO	owner:"172.26.104.123:2882", expired time:16:32:21.445173, ballot number:270	HAPPENED_TIME	2025-07-25 16:32:22.503836					
2025-07-25 08:51:22.325658	172.26.104.123	2882	ELECTION	witness change leader	TENANT_ID	1	LS_ID	1	INFO	witness change leader : "172.26.104.124:2882" -> "172.26.104.123:2882"	HAPPENED_TIME	2025-07-25 08:51:18.283614					
2025-07-25 08:51:22.321387	172.26.104.123	2882	ELECTION	change leader to takeover	TENANT_ID	1	LS_ID	1	INFO	new leader takeover : "172.26.104.124:2882" -> "172.26.104.123:2882"	HAPPENED_TIME	2025-07-25 08:51:18.179272					
2025-07-25 08:51:21.927138	172.26.104.123	2882	ELECTION	lease expired to revoke	TENANT_ID	1	LS_ID	1	INFO	leader lease expired, server|lease:[08:51:04.448|08:51:04.448|08:51:04.448]	HAPPENED_TIME	2025-07-25 08:51:06.10396					
2025-07-25 08:51:21.924816	172.26.104.123	2882	ELECTION	lease expired	TENANT_ID	1	LS_ID	1	INFO	owner:"172.26.104.123:2882", expired time:08:51:04.448956, ballot number:268	HAPPENED_TIME	2025-07-25 08:51:06.10086					
2025-07-25 08:51:20.494637	172.26.104.124	2882	ELECTION	witness change leader	TENANT_ID	1	LS_ID	1	INFO	witness change leader : "172.26.104.124:2882" -> "172.26.104.123:2882"	HAPPENED_TIME	2025-07-25 08:51:18.213192					
2025-07-25 08:51:18.213745	172.26.104.125	2882	ELECTION	witness change leader	TENANT_ID	1	LS_ID	1	INFO	witness change leader : "172.26.104.124:2882" -> "172.26.104.123:2882"	HAPPENED_TIME	2025-07-25 08:51:18.213677					
2025-07-25 08:51:18.173158	172.26.104.124	2882	ELECTION	change leader to revoke	TENANT_ID	1	LS_ID	1	INFO	old leader revoke : "172.26.104.124:2882" -> "172.26.104.123:2882"	HAPPENED_TIME	2025-07-25 08:51:18.173099					
2025-07-25 08:51:17.966377	172.26.104.124	2882	ELECTION	prepare change leader	TENANT_ID	1	LS_ID	1	INFO	leader prepare to change leader : "172.26.104.124:2882" -> "172.26.104.123:2882", reason : ZONE PRIORITY	HAPPENED_TIME	2025-07-25 08:51:17.966303					
2025-07-25 08:51:17.957757	172.26.104.124	2882	ELECTION	prepare change leader	TENANT_ID	1	LS_ID	1	INFO	leader prepare to change leader : "172.26.104.124:2882" -> "172.26.104.123:2882", reason : ZONE PRIORITY	HAPPENED_TIME	2025-07-25 08:51:17.957714					
2025-07-25 08:51:17.906197	172.26.104.124	2882	ELECTION	prepare change leader	TENANT_ID	1	LS_ID	1	INFO	leader prepare to change leader : "172.26.104.124:2882" -> "172.26.104.123:2882", reason : ZONE PRIORITY	HAPPENED_TIME	2025-07-25 08:51:17.906147					
2025-07-25 08:51:08.335647	172.26.104.124	2882	ELECTION	decentralized to be leader	TENANT_ID	1	LS_ID	1	INFO	server|lease:["172.26.104.124:2882"|2025-07-25 08:51:08.465,"172.26.104.125:2882"|2025-07-25 08:51:08.465]	HAPPENED_TIME	2025-07-25 08:51:04.466026					
2025-07-25 08:51:08.333520	172.26.104.124	2882	ELECTION	vote	TENANT_ID	1	LS_ID	1	INFO	vote for "172.26.104.124:2882", reason:IP-PORT(priority equal)	HAPPENED_TIME	2025-07-25 08:51:04.463080					
2025-07-25 08:51:08.331440	172.26.104.124	2882	ELECTION	lease expired	TENANT_ID	1	LS_ID	1	INFO	owner:"172.26.104.123:2882", expired time:08:51:04.449241, ballot number:268	HAPPENED_TIME	2025-07-25 08:51:04.462922					
2025-07-25 08:51:08.072189	172.26.104.125	2882	ELECTION	vote	TENANT_ID	1	LS_ID	1	INFO	vote for "172.26.104.124:2882", reason:IP-PORT(priority equal)	HAPPENED_TIME	2025-07-25 08:51:04.465420					
2025-07-25 08:51:08.069470	172.26.104.125	2882	ELECTION	lease expired	TENANT_ID	1	LS_ID	1	INFO	owner:"172.26.104.123:2882", expired time:08:51:04.448864, ballot number:268	HAPPENED_TIME	2025-07-25 08:51:04.465280					
2025-07-25 06:56:33.992131	172.26.104.123	2882	ELECTION	witness change leader	TENANT_ID	1	LS_ID	1	INFO	witness change leader : "172.26.104.124:2882" -> "172.26.104.123:2882"	HAPPENED_TIME	2025-07-25 06:56:33.992076					
2025-07-25 06:56:33.739021	172.26.104.124	2882	ELECTION	witness change leader	TENANT_ID	1	LS_ID	1	INFO	witness change leader : "172.26.104.124:2882" -> "172.26.104.123:2882"	HAPPENED_TIME	2025-07-25 06:56:30.861934					
2025-07-25 06:56:33.732215	172.26.104.124	2882	ELECTION	change leader to revoke	TENANT_ID	1	LS_ID	1	INFO	old leader revoke : "172.26.104.124:2882" -> "172.26.104.123:2882"	HAPPENED_TIME	2025-07-25 06:56:30.824046					
2025-07-25 06:56:33.728373	172.26.104.124	2882	ELECTION	prepare change leader	TENANT_ID	1	LS_ID	1	INFO	leader prepare to change leader : "172.26.104.124:2882" -> "172.26.104.123:2882", reason : ZONE PRIORITY	HAPPENED_TIME	2025-07-25 06:56:30.822019