rootservice.log:[2024-09-24 13:31:33.149245] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720C-0-0] [lt=21] finish one round of loop in do_one_round_major_merge(expected_epoch=8, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 13:31:33.149307] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=53] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 13:31:33.149317] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=10] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 13:31:33.150127] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=36] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1726795251418738, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 13:31:33.150201] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=68] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1726795251421700, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 13:31:33.150235] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=32] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=880, zone_list=["zone1"]) rootservice.log:[2024-09-24 13:31:33.150259] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=19][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 13:31:33.150277] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=17][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 13:31:33.150285] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=7][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 13:31:33.150294] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=7][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 13:31:33.150304] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=8][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=8) rootservice.log:[2024-09-24 13:31:33.150313] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=8][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=8) rootservice.log:[2024-09-24 13:31:33.150369] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=7] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727155893150321), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 13:31:33.150403] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=31][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] rootservice.log:[2024-09-24 13:41:33.151360] ERROR issue_dba_error (ob_log.cpp:1891) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=8][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_major_merge_scheduler.cpp", line_no=826, info="long time major freeze not finish, please check it") rootservice.log:[2024-09-24 13:41:33.151382] EDIAG [RS.COMPACTION] check_merge_interval_time (ob_major_merge_scheduler.cpp:826) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=21][errcode=0] long time major freeze not finish, please check it(ret=0, ret="OB_SUCCESS", global_last_merged_time=1719943608996818, global_merge_start_time=1726336804833801, max_merge_time=1726336804833801, now=1727156493150425, tenant_id=1004, is_merging=true, start_service_time=1726796148962920, total_service_time=360344187505) BACKTRACE:0x16bb358d 0x8185e12 0x8185961 0x81855c6 0x81853a7 0xa321d6d 0x5e9aaec 0xa307fed 0xa30367a 0x5e31fc4 0x16c3232c 0x16c3d541 0x16c398be 0x14c668165fdd 0x14c668096d0f rootservice.log:[2024-09-24 13:41:33.151409] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720D-0-0] [lt=25] finish one round of loop in do_one_round_major_merge(expected_epoch=8, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 13:41:33.151457] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=35] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 13:41:33.151469] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=11] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 13:41:33.152313] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=21] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1726795251418738, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 13:41:33.152353] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=35] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1726795251421700, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 13:41:33.152368] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=14] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=875, zone_list=["zone1"]) rootservice.log:[2024-09-24 13:41:33.152383] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=8][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 13:41:33.152392] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=8][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 13:41:33.152399] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=5][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 13:41:33.152406] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=5][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 13:41:33.152413] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=4][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=8) rootservice.log:[2024-09-24 13:41:33.152420] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=6][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=8) rootservice.log:[2024-09-24 13:41:33.152468] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=4] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727156493152426), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 13:41:33.152477] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=7][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=605, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 13:51:33.153152] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720E-0-0] [lt=6] finish one round of loop in do_one_round_major_merge(expected_epoch=8, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 13:51:33.153230] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=65] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 13:51:33.153244] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=12] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 13:51:33.154373] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=190] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1726795251418738, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 13:51:33.154437] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=59] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1726795251421700, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 13:51:33.154456] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=17] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1018, zone_list=["zone1"]) rootservice.log:[2024-09-24 13:51:33.154472] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=11][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 13:51:33.154479] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=7][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 13:51:33.154487] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=6][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 13:51:33.154495] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=6][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 13:51:33.154505] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=7][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=8) rootservice.log:[2024-09-24 13:51:33.154513] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=7][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=8) rootservice.log:[2024-09-24 13:51:33.154578] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=4] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727157093154519), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 13:51:33.154591] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=10][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=606, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 14:01:33.155191] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E298720F-0-0] [lt=5] finish one round of loop in do_one_round_major_merge(expected_epoch=8, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:01:33.155285] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=74] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 14:01:33.155301] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=14] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:01:33.156385] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=74] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1726795251418738, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:01:33.156443] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=53] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1726795251421700, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:01:33.156464] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=18] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1084, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:01:33.156488] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=14][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 14:01:33.156503] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=14][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:01:33.156517] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=12][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:01:33.156531] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=12][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 14:01:33.156548] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=12][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=8) rootservice.log:[2024-09-24 14:01:33.156564] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=14][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=8) rootservice.log:[2024-09-24 14:01:33.156724] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=64] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727157693156631), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 14:01:33.156749] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=20][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=607, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 14:11:33.157369] ERROR issue_dba_error (ob_log.cpp:1891) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=15][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_major_merge_scheduler.cpp", line_no=826, info="long time major freeze not finish, please check it") rootservice.log:[2024-09-24 14:11:33.157414] EDIAG [RS.COMPACTION] check_merge_interval_time (ob_major_merge_scheduler.cpp:826) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=43][errcode=0] long time major freeze not finish, please check it(ret=0, ret="OB_SUCCESS", global_last_merged_time=1719943608996818, global_merge_start_time=1726336804833801, max_merge_time=1726336804833801, now=1727158293156274, tenant_id=1004, is_merging=true, start_service_time=1726796148962920, total_service_time=362144193354) BACKTRACE:0x16bb358d 0x8185e12 0x8185961 0x81855c6 0x81853a7 0xa321d6d 0x5e9aaec 0xa307fed 0xa30367a 0x5e31fc4 0x16c3232c 0x16c3d541 0x16c398be 0x14c668165fdd 0x14c668096d0f rootservice.log:[2024-09-24 14:11:33.157440] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987210-0-0] [lt=25] finish one round of loop in do_one_round_major_merge(expected_epoch=8, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:11:33.157510] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=56] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 14:11:33.157564] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=53] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:11:33.158566] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=39] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1726795251418738, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:11:33.158643] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=70] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1726795251421700, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:11:33.158669] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=24] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1062, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:11:33.158703] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=28][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 14:11:33.158711] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=8][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:11:33.158719] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=6][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:11:33.158725] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=4][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 14:11:33.158732] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=5][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=8) rootservice.log:[2024-09-24 14:11:33.158738] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=5][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=8) rootservice.log:[2024-09-24 14:11:33.158813] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=27] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727158293158767), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 14:11:33.158832] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=17][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=608, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 14:21:33.159402] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987211-0-0] [lt=6] finish one round of loop in do_one_round_major_merge(expected_epoch=8, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:21:33.159462] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=50] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 14:21:33.159472] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=10] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:21:33.160382] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=26] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1726795251418738, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:21:33.160431] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=45] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1726795251421700, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:21:33.160446] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=14] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=945, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:21:33.160467] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=17][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 14:21:33.160477] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=10][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:21:33.160485] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=7][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:21:33.160491] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=5][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 14:21:33.160499] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=6][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=8) rootservice.log:[2024-09-24 14:21:33.160506] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=6][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=8) rootservice.log:[2024-09-24 14:21:33.160648] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=4] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727158893160511), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 14:21:33.160661] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=11][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=609, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 14:31:33.161547] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987212-0-0] [lt=5] finish one round of loop in do_one_round_major_merge(expected_epoch=8, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:31:33.161607] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=42] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 14:31:33.161635] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=27] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:31:33.162862] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=33] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1726795251418738, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:31:33.162918] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=49] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1726795251421700, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:31:33.162956] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=36] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1283, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:31:33.162994] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=24][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 14:31:33.163019] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=24][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:31:33.163032] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=10][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:31:33.163052] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=18][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 14:31:33.163065] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=10][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=8) rootservice.log:[2024-09-24 14:31:33.163087] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=20][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=8) rootservice.log:[2024-09-24 14:31:33.163183] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=9] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727159493163098), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 14:31:33.163217] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=28][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=610, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 14:41:33.164534] ERROR issue_dba_error (ob_log.cpp:1891) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=8][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_major_merge_scheduler.cpp", line_no=826, info="long time major freeze not finish, please check it") rootservice.log:[2024-09-24 14:41:33.164568] EDIAG [RS.COMPACTION] check_merge_interval_time (ob_major_merge_scheduler.cpp:826) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=33][errcode=0] long time major freeze not finish, please check it(ret=0, ret="OB_SUCCESS", global_last_merged_time=1719943608996818, global_merge_start_time=1726336804833801, max_merge_time=1726336804833801, now=1727160093163195, tenant_id=1004, is_merging=true, start_service_time=1726796148962920, total_service_time=363944200275) BACKTRACE:0x16bb358d 0x8185e12 0x8185961 0x81855c6 0x81853a7 0xa321d6d 0x5e9aaec 0xa307fed 0xa30367a 0x5e31fc4 0x16c3232c 0x16c3d541 0x16c398be 0x14c668165fdd 0x14c668096d0f rootservice.log:[2024-09-24 14:41:33.164613] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987213-0-0] [lt=43] finish one round of loop in do_one_round_major_merge(expected_epoch=8, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:41:33.164673] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=44] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 14:41:33.164684] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=10] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:41:33.165640] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=22] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1726795251418738, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:41:33.165696] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=50] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1726795251421700, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:41:33.165713] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=15] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1004, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:41:33.165751] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=9][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 14:41:33.165759] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=8][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:41:33.165767] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=6][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:41:33.165775] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=5][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 14:41:33.165783] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=5][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=8) rootservice.log:[2024-09-24 14:41:33.165799] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=14][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=8) rootservice.log:[2024-09-24 14:41:33.165862] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=5] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727160093165805), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 14:41:33.165892] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=27][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] rootservice.log:[2024-09-24 14:51:33.166935] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987214-0-0] [lt=7] finish one round of loop in do_one_round_major_merge(expected_epoch=8, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:51:33.166990] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=44] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 14:51:33.167002] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=11] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:51:33.167822] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=28] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1726795251418738, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:51:33.167856] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=30] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1726795251421700, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:8, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:51:33.167871] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=14] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=838, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:51:33.167883] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=8][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 14:51:33.167891] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=7][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:51:33.167897] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=5][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:51:33.167904] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=6][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 14:51:33.167911] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=5][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=8) rootservice.log:[2024-09-24 14:51:33.167918] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=6][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=8) rootservice.log:[2024-09-24 14:51:33.167969] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=4] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727160693167923), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 14:51:33.167979] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [1292069][T1004_MergeSche][T1004][YB420A8F51B1-00062282E2987215-0-0] [lt=8][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=612, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 14:53:01.340390] INFO [RS.COMPACTION] run3 (ob_major_merge_scheduler.cpp:133) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=4] major merge scheduler will run(tenant_id=1004) rootservice.log:[2024-09-24 14:53:01.364564] INFO [SHARE] update_service_epoch (ob_service_epoch_proxy.cpp:157) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=28] succ to update service epoch(tenant_id=1004, name="freeze_service_epoch", epoch_value=9, affected_rows=1) rootservice.log:[2024-09-24 14:53:01.364585] INFO [RS.COMPACTION] do_update_freeze_service_epoch (ob_major_merge_scheduler.cpp:653) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=18] REACH SYSLOG RATE LIMIT [bandwidth] rootservice.log:[2024-09-24 14:53:02.607091] INFO [RS.COMPACTION] reload (ob_zone_merge_manager.cpp:67) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=10] start to reload zone_merge_mgr(tenant_id=1004, is_loaded=true, global_merge_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}, zone_merge_infos=[{tenant_id:1004, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943609007630, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804863472, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) rootservice.log:[2024-09-24 14:53:02.608507] INFO [SHARE] load_global_merge_info (ob_global_merge_table_operator.cpp:90) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=155] finish load_gloal_merge_info(ret=0, ret="OB_SUCCESS", tenant_id=1004, sql=SELECT * FROM __all_merge_info WHERE tenant_id = '1004') rootservice.log:[2024-09-24 14:53:02.610340] INFO [SHARE] inner_load_zone_merge_infos_ (ob_zone_merge_table_operator.cpp:399) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=20] finish load_zone_merge_info(ret=0, ret="OB_SUCCESS", tenant_id=1004, sql=SELECT * FROM __all_zone_merge_info WHERE (tenant_id, zone) IN (('1004', 'zone1'))) rootservice.log:[2024-09-24 14:53:02.610426] INFO [RS.COMPACTION] reload (ob_zone_merge_manager.cpp:121) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=29] succ to reload zone merge manager(zone_list=["zone1"], global_merge_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}, zone_merge_infos=[{tenant_id:1004, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943609007630, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804863472, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) rootservice.log:[2024-09-24 14:53:02.612422] INFO [SHARE] get_freeze_info_larger_or_equal_than (ob_freeze_info_proxy.cpp:168) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=52] finish load_freeze_info(ret=0, ret="OB_SUCCESS", tenant_id=1004, sql=SELECT * FROM __all_freeze_info WHERE frozen_scn >= 1726336804517991694 ORDER BY frozen_scn) rootservice.log:[2024-09-24 14:53:02.612478] INFO [SHARE] update_freeze_info (ob_freeze_info_manager.cpp:219) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=14] inner load succ(latest_freeze_info={frozen_version:{val:1726336804517991694, v:0}, schema_version:1715939734621976, data_version:17180065793}, freeze_info_={latest_snapshot_gc_scn:{val:1727160708871014947, v:0}, frozen_statuses_.count():1, frozen_statuses:[{frozen_version:{val:1726336804517991694, v:0}, schema_version:1715939734621976, data_version:17180065793}]}) rootservice.log:[2024-09-24 14:53:02.612498] INFO [RS] reload (ob_major_merge_info_manager.cpp:94) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=18] succ to reload merge info manager(tenant_id=1004) rootservice.log:[2024-09-24 14:53:02.612507] INFO [RS.COMPACTION] try_update_epoch_and_reload (ob_major_merge_scheduler.cpp:627) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=6] succ to try_update_epoch_and_reload(ori_epoch=-1, latest_epoch=9) rootservice.log:[2024-09-24 14:53:02.612523] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:138) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=7] zone_merge_mgr is already loaded(tenant_id=1004, global_merge_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}, zone_merge_infos=[{tenant_id:1004, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943609007630, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804863472, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) rootservice.log:[2024-09-24 14:53:02.613301] INFO [RS.COMPACTION] clear_cached_info (ob_major_merge_progress_checker.cpp:152) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=46] success to clear cached info(ret=0, ret="OB_SUCCESS", tenant_id=1004, compaction_scn={val:18446744073709551615, v:3}) rootservice.log:[2024-09-24 14:53:02.614163] WDIAG [SQL.RESV] resolve_table_relation_factor (ob_dml_resolver.cpp:8242) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBC8-0-0] [lt=10][errcode=0] resolve dblink name(has_dblink_node=false, dblink_name=, is_reverse_link=false) rootservice.log:[2024-09-24 14:53:02.622439] INFO [STORAGE] check_status_for_new_mds (ob_tablet_create_delete_helper.cpp:195) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBC8-0-0] [lt=13] refresh tablet status cache(ret=0, ls_id={id:1}, tablet_id={id:373}, tablet_status_cache={tablet_status:{val:1, str:"NORMAL"}, create_commit_version:1712127877470362724, delete_commit_version:-1}, snapshot_version=4611686018427387895) rootservice.log:[2024-09-24 14:53:02.704005] INFO [STORAGE] check_snapshot_readable_with_cache (ob_tablet.cpp:7228) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBC8-0-0] [lt=22] refresh ddl data cache(ret=0, tablet_meta_.ls_id_={id:1}, tablet_id={id:373}, ddl_data_cache_={redefined:false, schema_version:0, snapshot_version:0}) rootservice.log:[2024-09-24 14:53:02.704570] INFO [RS.COMPACTION] set_basic_info (ob_major_merge_progress_checker.cpp:139) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=23] success to set basic info(ret=0, ret="OB_SUCCESS", compaction_scn={val:1726336804517991694, v:0}, expected_epoch=9) rootservice.log:[2024-09-24 14:53:02.705319] WDIAG [SQL.RESV] resolve_table_relation_factor (ob_dml_resolver.cpp:8242) [2150021][T1004_MergeSche][T1003][YB420A8F51B1-000622D80211FBC9-0-0] [lt=22][errcode=0] resolve dblink name(has_dblink_node=false, dblink_name=, is_reverse_link=false) rootservice.log:[2024-09-24 14:53:02.707339] INFO [STORAGE] check_status_for_new_mds (ob_tablet_create_delete_helper.cpp:195) [2150021][T1004_MergeSche][T1003][YB420A8F51B1-000622D80211FBC9-0-0] [lt=12] refresh tablet status cache(ret=0, ls_id={id:1}, tablet_id={id:416}, tablet_status_cache={tablet_status:{val:1, str:"NORMAL"}, create_commit_version:1712127867628982828, delete_commit_version:-1}, snapshot_version=4611686018427387895) rootservice.log:[2024-09-24 14:53:02.712725] INFO [STORAGE] check_snapshot_readable_with_cache (ob_tablet.cpp:7228) [2150021][T1004_MergeSche][T1003][YB420A8F51B1-000622D80211FBC9-0-0] [lt=31] refresh ddl data cache(ret=0, tablet_meta_.ls_id_={id:1}, tablet_id={id:416}, ddl_data_cache_={redefined:false, schema_version:0, snapshot_version:0}) rootservice.log:[2024-09-24 14:53:02.713120] INFO [SHARE] delete_column_checksum_err_info (ob_column_checksum_error_operator.cpp:119) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=17] succ to delete column checksum error info(tenant_id=1004, min_frozen_scn={val:1726336804517991694, v:0}, affected_rows=0) rootservice.log:[2024-09-24 14:53:02.713261] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:322) [2150021][T1004_MergeSche][T1004][Y0-0000000000000000-0-0] [lt=131] start to do one round major_merge(expected_epoch=9) rootservice.log:[2024-09-24 14:53:02.713285] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=9] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 14:53:02.713483] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=196] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:53:02.715178] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=49] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1727160773935107, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:53:02.715241] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=57] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1727160773935284, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:53:02.715267] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=23] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1715, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:53:02.715288] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=13][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 14:53:02.715298] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=9][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:53:02.715307] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=7][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:53:02.715316] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=6][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 14:53:02.715327] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=7][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=9) rootservice.log:[2024-09-24 14:53:02.715336] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=8][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=9) rootservice.log:[2024-09-24 14:53:02.715399] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=5] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727160782715343), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 14:53:02.715421] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:193) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=19][errcode=0] fail to major merge, will immediate retry(tenant_id=1004, fail_count=1, IMMEDIATE_RETRY_CNT=3, idle_time_us=10000000) rootservice.log:[2024-09-24 14:53:12.716665] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCA-0-0] [lt=7] finish one round of loop in do_one_round_major_merge(expected_epoch=9, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:53:12.716728] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=48] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 14:53:12.716739] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=11] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:53:12.717912] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=22] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1727160773935107, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:53:12.717952] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=37] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1727160773935284, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:53:12.717969] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=15] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1202, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:53:12.717984] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=9][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 14:53:12.717993] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=9][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:53:12.718001] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=6][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:53:12.718009] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=6][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 14:53:12.718017] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=5][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=9) rootservice.log:[2024-09-24 14:53:12.718025] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=7][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=9) rootservice.log:[2024-09-24 14:53:12.718032] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:193) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=4][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] rootservice.log:[2024-09-24 14:53:32.719664] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCB-0-0] [lt=7] finish one round of loop in do_one_round_major_merge(expected_epoch=9, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:53:32.719716] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=39] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 14:53:32.719725] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=9] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 14:53:32.720928] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=21] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1727160773935107, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:53:32.720965] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=32] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1727160773935284, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:53:32.720980] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=14] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1230, zone_list=["zone1"]) rootservice.log:[2024-09-24 14:53:32.720994] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=8][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 14:53:32.721002] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=7][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:53:32.721008] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=5][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 14:53:32.721016] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=6][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 14:53:32.721023] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=4][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=9) rootservice.log:[2024-09-24 14:53:32.721030] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=6][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=9) rootservice.log:[2024-09-24 14:53:32.721037] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=4][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=3, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 15:03:32.722733] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCC-0-0] [lt=5] finish one round of loop in do_one_round_major_merge(expected_epoch=9, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:03:32.722823] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=75] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 15:03:32.722837] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=13] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:03:32.723920] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=35] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1727160773935107, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:03:32.723970] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=46] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1727160773935284, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:03:32.723987] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=15] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1110, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:03:32.724002] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=10][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 15:03:32.724018] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=15][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:03:32.724026] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=7][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:03:32.724039] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=12][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 15:03:32.724048] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=6][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=9) rootservice.log:[2024-09-24 15:03:32.724063] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=14][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=9) rootservice.log:[2024-09-24 15:03:32.724111] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=5] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727161412724069), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 15:03:32.724150] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=37][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=4, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 15:13:32.725790] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCD-0-0] [lt=6] finish one round of loop in do_one_round_major_merge(expected_epoch=9, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:13:32.725876] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=70] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 15:13:32.725889] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=12] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:13:32.727046] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=22] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1727160773935107, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:13:32.727130] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=77] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1727160773935284, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:13:32.727152] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=19] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1237, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:13:32.727169] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=10][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 15:13:32.727178] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=8][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:13:32.727187] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=6][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:13:32.727195] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=5][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 15:13:32.727203] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=5][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=9) rootservice.log:[2024-09-24 15:13:32.727211] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=7][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=9) rootservice.log:[2024-09-24 15:13:32.727290] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=6] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727162012727218), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 15:13:32.727312] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=18][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=5, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 15:23:32.729208] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCE-0-0] [lt=37] finish one round of loop in do_one_round_major_merge(expected_epoch=9, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:23:32.729289] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=67] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 15:23:32.729300] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=11] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:23:32.730353] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=22] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1727160773935107, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:23:32.730397] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=41] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1727160773935284, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:23:32.730413] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=14] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1087, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:23:32.730428] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=9][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 15:23:32.730438] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=9][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:23:32.730445] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=5][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:23:32.730452] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=5][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 15:23:32.730460] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=5][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=9) rootservice.log:[2024-09-24 15:23:32.730467] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=6][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=9) rootservice.log:[2024-09-24 15:23:32.730589] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=4] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727162612730472), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 15:23:32.730603] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=10][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] rootservice.log:[2024-09-24 15:33:32.731580] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBCF-0-0] [lt=7] finish one round of loop in do_one_round_major_merge(expected_epoch=9, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:33:32.731654] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=61] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 15:33:32.731667] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=12] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:33:32.732728] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=36] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1727160773935107, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:33:32.732781] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=49] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1727160773935284, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:33:32.732805] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=23] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1100, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:33:32.732819] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=9][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 15:33:32.732832] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=12][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:33:32.732842] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=9][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:33:32.732850] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=7][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 15:33:32.732873] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=22][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=9) rootservice.log:[2024-09-24 15:33:32.732885] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=11][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=9) rootservice.log:[2024-09-24 15:33:32.732935] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=10] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727163212732896), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 15:33:32.732948] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=12][errcode=0] major merge failed more than immediate cnt, turn to idle status(fail_count=7, IMMEDIATE_RETRY_CNT=3, idle_time_us=600000000) rootservice.log:[2024-09-24 15:43:32.733718] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:371) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD0-0-0] [lt=7] finish one round of loop in do_one_round_major_merge(expected_epoch=9, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:43:32.733833] INFO [RS.COMPACTION] get_next_zone (ob_tenant_all_zone_merge_strategy.cpp:36) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=101] get_next_zone of merge strategy(tenant_id_=1004, to_merge_cnt=0, to_merge_zones=[]) rootservice.log:[2024-09-24 15:43:32.733886] INFO [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:341) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=52] no more zone need to merge(tenant_id=1004, global_info={tenant_id:1004, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1719943200208502037, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1726336804833801, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:1719943608996818, need_update:false}}) rootservice.log:[2024-09-24 15:43:32.735161] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=45] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1727160773935107, create_time_us:1712127874158294, tenant_id:1004, ls_id:{id:1}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:43:32.735249] INFO [COMMON] refresh_by_zone (ob_compaction_locality_cache.cpp:187) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=77] success to refresh cached ls_info(ret=0, tmp_ls_info={tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1727160773935284, create_time_us:1712127887720096, tenant_id:1004, ls_id:{id:1001}, server:"10.143.81.177:2882", sql_port:2881, role:1, member_list:[{server:"10.143.81.177:2882", timestamp:1}], replica_type:0, proposal_id:9, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1002, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:43:32.735275] INFO [COMMON] inner_refresh_ls_locality (ob_compaction_locality_cache.cpp:126) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=25] finish to refresh ls locality cache(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_ts=1341, zone_list=["zone1"]) rootservice.log:[2024-09-24 15:43:32.735291] WDIAG [RS.COMPACTION] check_schema_version (ob_major_merge_progress_checker.cpp:269) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=10][errcode=-4023] is not a formal_schema_version(ret=-4023, ret="OB_EAGAIN", local_schema_version=1715939734621975) rootservice.log:[2024-09-24 15:43:32.735317] WDIAG [RS.COMPACTION] prepare_unfinish_table_ids (ob_major_merge_progress_checker.cpp:282) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=25][errcode=-4023] fail to check schema version(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:43:32.735326] WDIAG [RS.COMPACTION] prepare_check_progress (ob_major_merge_progress_checker.cpp:439) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=7][errcode=-4023] fail to prepare table_id_map(ret=-4023, ret="OB_EAGAIN", tenant_id=1004) rootservice.log:[2024-09-24 15:43:32.735333] WDIAG [RS.COMPACTION] check_progress (ob_major_merge_progress_checker.cpp:517) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=6][errcode=-4023] failed to prepare check progress(ret=-4023, ret="OB_EAGAIN") rootservice.log:[2024-09-24 15:43:32.735342] WDIAG [RS.COMPACTION] update_merge_status (ob_major_merge_scheduler.cpp:501) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=7][errcode=-4023] fail to check merge status(ret=-4023, ret="OB_EAGAIN", tenant_id=1004, expected_epoch=9) rootservice.log:[2024-09-24 15:43:32.735353] WDIAG [RS.COMPACTION] do_one_round_major_merge (ob_major_merge_scheduler.cpp:349) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=10][errcode=-4023] fail to update merge status(ret=-4023, ret="OB_EAGAIN", expected_epoch=9) rootservice.log:[2024-09-24 15:43:32.735445] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=23] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1727163812735378), 'daily_merge', 'merge_process', 'tenant_id', 1004, 'check merge progress fail', -4023, 'global_broadcast_scn', '{name:"global_broadcast_scn", is_scn:true, scn:{val:1726336804517991694, v:0}, value:-1, need_update:false}', 'service_addr', '"10.143.81.177:2882"', '', '', '10.143.81.177', 2882)) rootservice.log:[2024-09-24 15:43:32.735478] WDIAG [RS.COMPACTION] try_idle (ob_major_merge_scheduler.cpp:197) [2150021][T1004_MergeSche][T1004][YB420A8F51B1-000622D80211FBD1-0-0] [lt=29][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth]