[2025-03-19 15:02:39.290581] INFO [SQL.ENG] init_direct_load_ctx (ob_direct_load_optimizer_ctx.cpp:230) [785035][T1_BackupDataSr][T1][YB420A686AAD-00062F071CB1EC80-0-0] [lt=42] init direct load ctx result(ret=0, direct_load_hint={has_direct:0, need_sort:0, has_no_direct:0, flags:0, max_error_row_count:0, load_method:"INVALID_LOAD_METHOD"}, global_hint.has_append()=false, table_id_=18446744073709551615, load_method_=0, insert_mode_=0, load_mode_=0, load_level_=0, dup_action_=3, max_error_row_count_=0, need_sort_=false, can_use_direct_load_=false, use_direct_load_=false, is_optimized_by_default_load_mode_=false) [2025-03-19 15:02:39.292559] INFO [STORAGE] create_memtable (ob_protected_memtable_mgr_handle.h:158) [785035][T1_BackupDataSr][T1][YB420A686AAD-00062F071CB1EC80-0-0] [lt=49] memtable_mgr_handle_ is not exist, need create(tablet_meta.ls_id_={id:1}, tablet_meta.tablet_id_={id:358}) [2025-03-19 15:02:39.292597] INFO [STORAGE] init (ob_medium_compaction_mgr.cpp:156) [785035][T1_BackupDataSr][T1][YB420A686AAD-00062F071CB1EC80-0-0] [lt=31] success to init medium clog recorder(ret=0, ls_id={id:1}, tablet_id={id:358}, max_saved_version=0) [2025-03-19 15:02:39.292618] INFO [STORAGE.TRANS] init_storage_recorder (ob_tablet_memtable_mgr.cpp:144) [785035][T1_BackupDataSr][T1][YB420A686AAD-00062F071CB1EC80-0-0] [lt=20] success to init storage recorder(ret=0, ls_id={id:1}, tablet_id={id:358}, max_saved_schema_version=0, max_saved_medium_scn=0, compat_mode=0) [2025-03-19 15:02:39.292652] INFO [STORAGE] create_memtable_ (ob_tablet_memtable_mgr.cpp:346) [785035][T1_BackupDataSr][T1][YB420A686AAD-00062F071CB1EC80-0-0] [lt=19] succeed to create memtable(arg={schema_version_:0, clog_checkpoint_scn_:{val:1742320865531816000, v:0}, new_clog_checkpoint_scn_:{val:1742320865531816000, v:0}, for_replay_:false, for_inc_direct_load_:false}, ret=0, ls_id={id:1}, new_tablet_memtable={ObITabletMemtable:{ObITable:{this:0x7effbc1e1930, key:{tablet_id:{id:358}, column_group_idx:0, table_type:"MEMTABLE", scn_range:{start_scn:{val:1742320865531816000, v:0}, end_scn:{val:4611686018427387903, v:0}}}, ref_cnt:2, upper_trans_version:9223372036854775807, timestamp:1742367759292647}, ls_id_:{id:1}, allow_freeze_:true, is_flushed_:false, is_tablet_freeze_:false, logging_blocked_:false, resolved_active_memtable_left_boundary_:true, unset_active_memtable_logging_blocked_:false, has_backoffed_:false, offlined_:false, read_barrier_:false, freeze_clock_:22, freeze_state_:1, unsubmitted_cnt_:0, init_timestamp_:1742367759292647, max_schema_version_:0, write_ref_cnt_:0, max_end_scn_:{val:0, v:0}, rec_scn_:{val:4611686018427387903, v:0}, freeze_scn_:{val:4611686018427387903, v:0}, freezer_:0x7effbc9e9010, memtable_mgr_handle_:{memtable_mgr:0x7effa6a22800, pool:0x7effd0580030}, mt_stat_.frozen_time_:0, mt_stat_.ready_for_flush_time_:0, mt_stat_.create_flush_dag_time_:0, mt_stat_.release_time_:0, mt_stat_.push_table_into_gc_queue_time_:0, mt_stat_.last_print_time_:0}, this:0x7effbc1e1930, state:0, max_data_schema_version:0, max_column_cnt:0, local_allocator:{ListHandle:{freeze_stat:0, id:-1, clock:9223372036854775807}, host:0x7effd73d62b0, arena_handle:{allocated:0}}, contain_hotspot_row:false, snapshot_version:{val:4611686018427387903, v:0}, contain_hotspot_row:false, ls_id:{id:1}, transfer_freeze_flag:false, recommend_snapshot_version:{val:18446744073709551615, v:3}}, this={ObIMemtableMgr:{Memtables:this:0x7effa6a22800, ref_cnt:2, is_inited:true, tablet_id:{id:358}, freezer:0x7effbc9e9010, memtable_head:0, memtable_tail:1, t3m:0x7effd72a6030, tables:[0x7effbc1e1930, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null]}, schema_recorder:{ObIStorageClogRecorder:{max_saved_version_:0, clog_scn_:{val:18446744073709551615, v:3}, log_handler_:0x7effbc9e3d10}, ls_id:{id:1}, tablet_id:{id:358}}}) [2025-03-19 15:02:39.292774] INFO [STORAGE] inner_create_memtable (ob_tablet.cpp:5194) [785035][T1_BackupDataSr][T1][YB420A686AAD-00062F071CB1EC80-0-0] [lt=117] succeeded to create memtable for tablet(ret=0, tablet_id={id:358}, ls_id={id:1}, schema_version=0, for_replay=false, for_direct_load=false, input_clog_checkpoint_scn={val:1742320865531816000, v:0}, latest_clog_checkpoint_scn={val:1742320865531816000, v:0}) [2025-03-19 15:02:39.292798] INFO [STORAGE] create_memtable (ob_tablet.cpp:5140) [785035][T1_BackupDataSr][T1][YB420A686AAD-00062F071CB1EC80-0-0] [lt=17] table store update memtable success(ret=0, ls_id={id:1}, tablet_id={id:358}, table_store_addr={ptr:0x7eff6e4b86f8, addr:{[ver=1,mode=0,seq=149800][2nd=148775][3rd=0][offset=581632,size=451,type=2,seq=1][trans_seq=0, sec_part=0][6th=1]}}, this=0x7eff6e4b80a0) [2025-03-19 15:02:39.297598] INFO [SHARE] move_job_to_his (ob_backup_data_table_operator.cpp:1036) [785035][T1_BackupDataSr][T1][YB420A686AAD-00062F071CB1EC80-0-0] [lt=23] [DATA_BACKUP]succeed move backup job to history table(tenant_id=1, job_id=2) [2025-03-19 15:02:39.297628] INFO [RS] move_to_history_ (ob_backup_data_scheduler.cpp:1844) [785035][T1_BackupDataSr][T1][YB420A686AAD-00062F071CB1EC80-0-0] [lt=28] sys tenant backup job succeed move to history table(job_attr_={job_id:2, tenant_id:1, incarnation_id:1, backup_set_id:0, initiator_tenant_id:1, initiator_job_id:2, executor_tenant_id:[1004], plus_archivelog:false, backup_level:{level:0}, backup_type:{type:1}, encryption_mode:0, passwd:"", backup_path:"", description:"ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2025-03-19 15:02:09.0", start_ts:1742367729204522, end_ts:1742367759281996, status:{status:3}, result:-9120, can_retry:true, retry_count:0, comment:""}) [2025-03-19 15:02:39.376720] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.104.106.173:2882") [2025-03-19 15:02:39.376762] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=42] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.104.106.174:2882") [2025-03-19 15:02:39.376773] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.104.106.175:2882") [2025-03-19 15:02:39.377222] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D9585FE-0-0] [lt=74] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476753765000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476766497000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476772834000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:39.380568] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:468) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D9585FE-0-0] [lt=129] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-03-19 15:02:39.380601] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D9585FE-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:39.544298] INFO [RS] do_work (ob_common_ls_service.cpp:138) [785027][T1_COMMONLSSe][T1][YB420A686AAD-00062F0999ED92E8-0-0] [lt=39] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:39.666633] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C4-0-0] [lt=20] is primary tenant, no need check readable_scn [2025-03-19 15:02:39.671953] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C4-0-0] [lt=33] is primary tenant, no need check readable_scn [2025-03-19 15:02:39.674946] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C4-0-0] [lt=25] is primary tenant, no need check readable_scn [2025-03-19 15:02:39.675112] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C4-0-0] [lt=22] is primary tenant, no need check readable_scn [2025-03-19 15:02:39.715384] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [784666][LeaseQueueTh2][T0][YB420A686AAE-00062F0805AA0A4B-0-0] [lt=47] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2025-03-19 15:02:39.769255] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [802099][T1003_StandbySc][T1003][YB420A686AAD-00062F0A7D35A98A-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:39.769264] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [801850][T1004_StandbySc][T1004][YB420A686AAD-00062F0A7D25A98B-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:39.777545] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1247) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57410-0-0] [lt=24] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-03-19 15:02:39.780906] INFO [RS] do_work (ob_common_ls_service.cpp:138) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57410-0-0] [lt=31] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:39.870168] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [785064][T1_HBService][T1][YB420A686AAD-00062F071C8EC504-0-0] [lt=39] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:39.875196] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98030][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594EBA-0-0] [lt=45] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2025-03-19 15:02:40.154017] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [785063][T1_HBService][T1][YB420A686AAD-00062F071C9ECF96-0-0] [lt=31] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:40.283511] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [785026][T1_StandbySchem][T1][YB420A686AAD-00062F09999DA081-0-0] [lt=34] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:40.314693] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98018][T1_L0_G0][T1][YB420A686AAF-00062F082D89D932-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2025-03-19 15:02:40.341496] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [784665][LeaseQueueTh1][T0][YB420A686AAD-00062F09A0AD65F1-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=23, ddl_arg=NULL) [2025-03-19 15:02:40.376979] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.104.106.173:2882") [2025-03-19 15:02:40.377016] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=37] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.104.106.174:2882") [2025-03-19 15:02:40.377027] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.104.106.175:2882") [2025-03-19 15:02:40.377044] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=10] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=419, return_code_array=[0, 0, 0], broadcast_times=1) [2025-03-19 15:02:40.384240] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D9585FF-0-0] [lt=73] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476753765000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476766497000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476772834000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:40.387714] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:468) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D9585FF-0-0] [lt=148] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-03-19 15:02:40.387744] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D9585FF-0-0] [lt=30] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:40.546316] INFO [RS] do_work (ob_common_ls_service.cpp:138) [785027][T1_COMMONLSSe][T1][YB420A686AAD-00062F0999ED92E9-0-0] [lt=34] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:40.645950] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [784664][LeaseQueueTh0][T0][YB420A686AAF-00062F07FAA915A1-0-0] [lt=35] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2025-03-19 15:02:40.676609] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C5-0-0] [lt=17] is primary tenant, no need check readable_scn [2025-03-19 15:02:40.682740] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C5-0-0] [lt=36] is primary tenant, no need check readable_scn [2025-03-19 15:02:40.686152] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C5-0-0] [lt=29] is primary tenant, no need check readable_scn [2025-03-19 15:02:40.686351] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C5-0-0] [lt=26] is primary tenant, no need check readable_scn [2025-03-19 15:02:40.769353] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [802099][T1003_StandbySc][T1003][YB420A686AAD-00062F0A7D35A98B-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:40.769355] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [801850][T1004_StandbySc][T1004][YB420A686AAD-00062F0A7D25A98C-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:40.788052] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1247) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57411-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-03-19 15:02:40.791344] INFO [RS] do_work (ob_common_ls_service.cpp:138) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57411-0-0] [lt=31] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:40.993814] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [802303][T1003_FrzInfoDe][T1003][YB420A686AAD-00062F087062F7A0-0-0] [lt=31] get number(ret=0, service_type_=0, range=1, base_id=1742367760993433000, start_id=1742367760993433000, end_id=1742367760993433001) [2025-03-19 15:02:40.995048] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:361) [802303][T1003_FrzInfoDe][T1003][YB420A686AAD-00062F087062F7A0-0-0] [lt=32] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1742367760993433000, v:0}, tenant_id=1003) [2025-03-19 15:02:41.112420] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [801944][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=31] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1742367760372270, tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1742367758493705000, replayable_scn:1742367758493705000, readable_scn:1742367758493705000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-03-19 15:02:41.256817] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [785286][T1_FrzInfoDet][T1][YB420A686AAD-00062F071C63C362-0-0] [lt=33] get number(ret=0, service_type_=0, range=1, base_id=1742367761256107000, start_id=1742367761256107000, end_id=1742367761256107001) [2025-03-19 15:02:41.258234] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:361) [785286][T1_FrzInfoDet][T1][YB420A686AAD-00062F071C63C362-0-0] [lt=34] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1742367761256107000, v:0}, tenant_id=1) [2025-03-19 15:02:41.283610] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [785026][T1_StandbySchem][T1][YB420A686AAD-00062F09999DA082-0-0] [lt=34] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:41.376436] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=22] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1742367759494844000, replayable_scn:1742367759494844000, readable_scn:1742367759494844000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1742367761373162, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1742367759494844000, replayable_scn:1742367759494844000, readable_scn:1742367759494844000, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1742367761373162, ora_rowscn_=1742367760680970000, content_changed=true) [2025-03-19 15:02:41.376815] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=38] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.104.106.173:2882") [2025-03-19 15:02:41.376839] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.104.106.174:2882") [2025-03-19 15:02:41.376848] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.104.106.175:2882") [2025-03-19 15:02:41.391454] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958600-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476753765000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476766497000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476772834000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:41.394937] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:468) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958600-0-0] [lt=136] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-03-19 15:02:41.394972] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958600-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:41.548103] INFO [RS] do_work (ob_common_ls_service.cpp:138) [785027][T1_COMMONLSSe][T1][YB420A686AAD-00062F0999ED92EA-0-0] [lt=32] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:41.595220] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [784757][DBMS_JOB_MASTER][T0][YB420A686AAD-00062F071AF100C6-0-0] [lt=17] dbms job master wait timeout, no entry(ret=-4018) [2025-03-19 15:02:41.596826] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [784757][DBMS_JOB_MASTER][T0][YB420A686AAD-00062F071AF100C6-0-0] [lt=30] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1742367761593461, delay:5000000, job_id:0, tenant_id:1004}, job_key=0x7effaa41a160, job_infos=[]) [2025-03-19 15:02:41.596868] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [784757][DBMS_JOB_MASTER][T0][YB420A686AAD-00062F071AF100C6-0-0] [lt=39] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1742367766596865, delay:5000000, job_id:0, tenant_id:1004}) [2025-03-19 15:02:41.687725] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C6-0-0] [lt=22] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"10.104.106.173:2882", readable_scn:{val:1742367761498578000, v:0}}, {server:"10.104.106.174:2882", readable_scn:{val:1742367761498578000, v:0}}, {server:"10.104.106.175:2882", readable_scn:{val:1742367761498578000, v:0}}], last_dump_ts_=1742367756635753, force_dump=false) [2025-03-19 15:02:41.687772] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C6-0-0] [lt=43] is primary tenant, no need check readable_scn [2025-03-19 15:02:41.692183] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C6-0-0] [lt=29] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1004, ls_id:{id:1}, sync_scn:{val:1742367761498578000, v:0}, readable_scn:{val:1742367761498578000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:1, config_seq:2}}) [2025-03-19 15:02:41.692973] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C6-0-0] [lt=39] is primary tenant, no need check readable_scn [2025-03-19 15:02:41.695883] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C6-0-0] [lt=21] is primary tenant, no need check readable_scn [2025-03-19 15:02:41.696234] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C6-0-0] [lt=25] is primary tenant, no need check readable_scn [2025-03-19 15:02:41.724582] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [784666][LeaseQueueTh2][T0][YB420A686AAE-00062F0805DA0965-0-0] [lt=33] [DDL] execute ddl like stmt(ret=0, cost=17, ddl_arg=NULL) [2025-03-19 15:02:41.769450] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [802099][T1003_StandbySc][T1003][YB420A686AAD-00062F0A7D35A98C-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:41.769454] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [801850][T1004_StandbySc][T1004][YB420A686AAD-00062F0A7D25A98D-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:41.798429] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1247) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57412-0-0] [lt=24] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-03-19 15:02:41.801652] INFO [RS] do_work (ob_common_ls_service.cpp:138) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57412-0-0] [lt=31] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:41.876757] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [785064][T1_HBService][T1][YB420A686AAD-00062F071C8EC505-0-0] [lt=27] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:42.084803] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [784757][DBMS_JOB_MASTER][T0][YB420A686AAD-00062F071AF100C6-0-0] [lt=21] dbms job master wait timeout, no entry(ret=-4018) [2025-03-19 15:02:42.086451] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [784757][DBMS_JOB_MASTER][T0][YB420A686AAD-00062F071AF100C6-0-0] [lt=33] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1742367762084687, delay:5000000, job_id:0, tenant_id:1003}, job_key=0x7effaa41a120, job_infos=[]) [2025-03-19 15:02:42.086505] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [784757][DBMS_JOB_MASTER][T0][YB420A686AAD-00062F071AF100C6-0-0] [lt=51] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1742367767086502, delay:5000000, job_id:0, tenant_id:1003}) [2025-03-19 15:02:42.086534] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [784757][DBMS_JOB_MASTER][T0][YB420A686AAD-00062F071AF100C6-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1742367767086529, delay:5000000, job_id:0, tenant_id:1004}) [2025-03-19 15:02:42.089257] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:115) [784753][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=26] start checking lost replicas [2025-03-19 15:02:42.146007] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98018][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594EC1-0-0] [lt=42] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2025-03-19 15:02:42.154513] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [785063][T1_HBService][T1][YB420A686AAD-00062F071C9ECF97-0-0] [lt=30] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:42.202087] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98032][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B35-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2025-03-19 15:02:42.264298] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [96652][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B36-0-0] [lt=49] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2025-03-19 15:02:42.283714] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [785026][T1_StandbySchem][T1][YB420A686AAD-00062F09999DA083-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:42.324483] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98030][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594EC3-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=12, ddl_arg=NULL) [2025-03-19 15:02:42.342106] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [784665][LeaseQueueTh1][T0][YB420A686AAD-00062F09A0AD65F2-0-0] [lt=33] [DDL] execute ddl like stmt(ret=0, cost=22, ddl_arg=NULL) [2025-03-19 15:02:42.377020] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.104.106.173:2882") [2025-03-19 15:02:42.377046] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.104.106.174:2882") [2025-03-19 15:02:42.377056] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.104.106.175:2882") [2025-03-19 15:02:42.398660] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958601-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476753765000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476766497000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476772834000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:42.402051] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:468) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958601-0-0] [lt=150] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-03-19 15:02:42.402085] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958601-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:42.416458] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98018][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594EC4-0-0] [lt=36] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2025-03-19 15:02:42.417644] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98025][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594EC5-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2025-03-19 15:02:42.440142] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98021][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B38-0-0] [lt=31] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2025-03-19 15:02:42.550097] INFO [RS] do_work (ob_common_ls_service.cpp:138) [785027][T1_COMMONLSSe][T1][YB420A686AAD-00062F0999ED92EB-0-0] [lt=35] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:42.569857] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [785208][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594EC6-0-0] [lt=36] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2025-03-19 15:02:42.570284] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [96648][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594EC7-0-0] [lt=34] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2025-03-19 15:02:42.607800] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98018][T1_L0_G0][T1][YB420A686AAF-00062F082D89D934-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2025-03-19 15:02:42.646600] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [784664][LeaseQueueTh0][T0][YB420A686AAF-00062F07FB591BDF-0-0] [lt=31] [DDL] execute ddl like stmt(ret=0, cost=22, ddl_arg=NULL) [2025-03-19 15:02:42.697709] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C7-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-03-19 15:02:42.703156] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C7-0-0] [lt=35] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"10.104.106.173:2882", readable_scn:{val:1742367762557178000, v:0}}, {server:"10.104.106.174:2882", readable_scn:{val:1742367762557178001, v:0}}, {server:"10.104.106.175:2882", readable_scn:{val:1742367762557178001, v:0}}], last_dump_ts_=1742367757651518, force_dump=false) [2025-03-19 15:02:42.703199] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C7-0-0] [lt=39] is primary tenant, no need check readable_scn [2025-03-19 15:02:42.706465] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C7-0-0] [lt=10] is primary tenant, no need check readable_scn [2025-03-19 15:02:42.706663] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C7-0-0] [lt=26] is primary tenant, no need check readable_scn [2025-03-19 15:02:42.706750] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C7-0-0] [lt=26] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1742367760496758000, v:0}) [2025-03-19 15:02:42.759592] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98018][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594EC8-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2025-03-19 15:02:42.769561] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [801850][T1004_StandbySc][T1004][YB420A686AAD-00062F0A7D25A98E-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:42.769560] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [802099][T1003_StandbySc][T1003][YB420A686AAD-00062F0A7D35A98D-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:42.808965] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1247) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57413-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-03-19 15:02:42.812265] INFO [RS] do_work (ob_common_ls_service.cpp:138) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57413-0-0] [lt=29] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:42.814548] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [803317][T1004_FrzInfoDe][T1004][YB420A686AAD-00062F0872C2F79E-0-0] [lt=33] get number(ret=0, service_type_=0, range=1, base_id=1742367762813532000, start_id=1742367762813532000, end_id=1742367762813532001) [2025-03-19 15:02:42.815881] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:361) [803317][T1004_FrzInfoDe][T1004][YB420A686AAD-00062F0872C2F79E-0-0] [lt=36] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1742367762813532000, v:0}, tenant_id=1004) [2025-03-19 15:02:42.916141] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98018][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B39-0-0] [lt=28] [DDL] execute ddl like stmt(ret=0, cost=6, ddl_arg=NULL) [2025-03-19 15:02:42.916625] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98028][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B3A-0-0] [lt=10] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2025-03-19 15:02:42.933624] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98025][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594EC9-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2025-03-19 15:02:42.933977] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [96661][T1_L0_G0][T1][YB420A686AAD-00062F0A9D594ECA-0-0] [lt=32] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2025-03-19 15:02:42.971229] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98018][T1_L0_G0][T1][YB420A686AAF-00062F082D89D935-0-0] [lt=32] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2025-03-19 15:02:43.125932] INFO [RS] recover_task (ob_ddl_scheduler.cpp:2515) [784637][TimerWK1][T0][Y0-0000000000000000-0-0] [lt=30] start processing ddl recovery(ddl_event_info={addr_:"10.104.106.173:2882", event_ts_:1742367763125931, sub_id_:0, trace_id_:Y0-0000000000000000-0-0, parent_trace_id_:Y0-0000000000000000-0-0}, task_records=[], primary_tenant_ids=[1004]) [2025-03-19 15:02:43.140988] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98032][T1_L0_G0][T1][YB420A686AAF-00062F082D89D936-0-0] [lt=31] [DDL] execute ddl like stmt(ret=0, cost=6, ddl_arg=NULL) [2025-03-19 15:02:43.190031] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98023][T1_L0_G0][T1][YB420A686AAF-00062F082D89D938-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2025-03-19 15:02:43.207242] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98029][T1_L0_G0][T1][YB420A686AAF-00062F082D89D939-0-0] [lt=48] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2025-03-19 15:02:43.283811] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [785026][T1_StandbySchem][T1][YB420A686AAD-00062F09999DA084-0-0] [lt=34] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:43.375376] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE60-0-0] [lt=28] root balance idle(idle_time=10000000) [2025-03-19 15:02:43.375434] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=46] start try disaster recovery [2025-03-19 15:02:43.375450] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2025-03-19 15:02:43.376947] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.104.106.173:2882") [2025-03-19 15:02:43.376973] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=25] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.104.106.174:2882") [2025-03-19 15:02:43.376982] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.104.106.175:2882") [2025-03-19 15:02:43.377002] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=14] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1004, cost_time_us=378, return_code_array=[0, 0, 0], broadcast_times=1) [2025-03-19 15:02:43.377167] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-03-19 15:02:43.378789] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=33] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1004, epoch_=1, all_service_names_=[{tenant_id:1004, service_name_id:1, service_name_str:{str:"ugs_lls_fas"}, service_status:1, service_status_str:"STARTED"}]) [2025-03-19 15:02:43.389436] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=40] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.173:2882", server_info_in_table={server:"10.104.106.173:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557429500476, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.390918] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=53] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.174:2882", server_info_in_table={server:"10.104.106.174:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557432223610, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.392387] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=36] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.175:2882", server_info_in_table={server:"10.104.106.175:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557431257473, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.392431] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3652) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=40] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2025-03-19 15:02:43.392462] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3948) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.392480] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4515) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1740557488605724, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.104.106.173:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1740557430262716, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1740557429284544, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1001, unit_group_id:1, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1002, unit_group_id:1, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1003, unit_group_id:1, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1001, unit_group_id:1, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1002, unit_group_id:1, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1003, unit_group_id:1, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-03-19 15:02:43.392674] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=188] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2025-03-19 15:02:43.392705] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.392731] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=25] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.392748] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.392768] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1740557488605724, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.104.106.173:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:2, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1001, unit_group_id:1, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1001, unit_group_id:1, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1740557430262716, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1002, unit_group_id:1, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:2, in_pool:true, unit:unit_id:2, resource_pool_id:1002, unit_group_id:1, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1740557429284544, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1003, unit_group_id:1, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:3, in_pool:true, unit:unit_id:3, resource_pool_id:1003, unit_group_id:1, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-03-19 15:02:43.392912] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4567) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=130] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.392939] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4711) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.392956] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5433) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.392973] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5722) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.394463] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=14] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2025-03-19 15:02:43.394486] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=21] start try tenant disaster recovery(tenant_id=1003, only_for_display=false) [2025-03-19 15:02:43.396081] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-03-19 15:02:43.405781] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958602-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476753765000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476766497000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476772834000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:43.408136] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=37] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.173:2882", server_info_in_table={server:"10.104.106.173:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557429500476, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.409116] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:468) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958602-0-0] [lt=134] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-03-19 15:02:43.409151] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958602-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:43.409617] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=42] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.174:2882", server_info_in_table={server:"10.104.106.174:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557432223610, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.411075] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=40] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.175:2882", server_info_in_table={server:"10.104.106.175:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557431257473, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.411119] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3652) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=40] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1003, ls_id={id:1}, acc_dr_task=0) [2025-03-19 15:02:43.411161] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3948) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=30] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.411183] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4515) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=20] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1003, ls_id:{id:1}, replicas:[{modify_time_us:1741079444384952, create_time_us:1741079438901707, tenant_id:1003, ls_id:{id:1}, server:"10.104.106.173:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079439529113, create_time_us:1741079438900313, tenant_id:1003, ls_id:{id:1}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079439454228, create_time_us:1741079438875441, tenant_id:1003, ls_id:{id:1}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1003, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-03-19 15:02:43.411369] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=181] build ls locality stat map(tenant_id=1003, ls_id={id:1}) [2025-03-19 15:02:43.411391] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.411414] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=22] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.411431] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.411450] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1741079444384952, create_time_us:1741079438901707, tenant_id:1003, ls_id:{id:1}, server:"10.104.106.173:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079439529113, create_time_us:1741079438900313, tenant_id:1003, ls_id:{id:1}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079439454228, create_time_us:1741079438875441, tenant_id:1003, ls_id:{id:1}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-03-19 15:02:43.411593] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4567) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=130] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.411620] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4711) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.411637] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5433) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.411653] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5722) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.413249] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=14] try tenant disaster recovery succ(tenant_id=1003, only_for_display=false) [2025-03-19 15:02:43.414977] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=23] start try tenant disaster recovery(tenant_id=1004, only_for_display=false) [2025-03-19 15:02:43.416648] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=30] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-03-19 15:02:43.426799] INFO [SQL.PC] need_check_schema_version (ob_plan_cache_value.cpp:2334) [784754][RootBalance][T1][YB420A686AAD-00062F0A2226BE61-0-0] [lt=40] need check schema(new_schema_version=1741079476939168, cached_tenant_schema_version=1741079476939168, is_contain_synonym()=false, contain_sys_name_table_=true, is_contain_tmp_tbl()=false, is_contain_sys_pl_object()=false, pc_ctx.sql_ctx_.session_info_->get_has_temp_table_flag()=false, need_check=true, constructed_sql_=SELECT * from __all_unit where resource_pool_id in (select resource_pool_id from __all_resource_pool where tenant_id = ?)) [2025-03-19 15:02:43.428859] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=43] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.173:2882", server_info_in_table={server:"10.104.106.173:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557429500476, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.430634] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=46] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.174:2882", server_info_in_table={server:"10.104.106.174:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557432223610, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.432123] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=42] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.175:2882", server_info_in_table={server:"10.104.106.175:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557431257473, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.432169] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3652) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=44] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1}, acc_dr_task=0) [2025-03-19 15:02:43.432200] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3948) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.432220] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4515) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1}, replicas:[{modify_time_us:1741079464410230, create_time_us:1741079458984469, tenant_id:1004, ls_id:{id:1}, server:"10.104.106.173:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079460420992, create_time_us:1741079458982413, tenant_id:1004, ls_id:{id:1}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079460353496, create_time_us:1741079458958460, tenant_id:1004, ls_id:{id:1}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-03-19 15:02:43.432403] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=179] build ls locality stat map(tenant_id=1004, ls_id={id:1}) [2025-03-19 15:02:43.432425] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.432448] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=22] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.432465] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.432484] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1741079464410230, create_time_us:1741079458984469, tenant_id:1004, ls_id:{id:1}, server:"10.104.106.173:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079460420992, create_time_us:1741079458982413, tenant_id:1004, ls_id:{id:1}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079460353496, create_time_us:1741079458958460, tenant_id:1004, ls_id:{id:1}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-03-19 15:02:43.432629] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4567) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=131] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.432655] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4711) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.432672] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5433) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.432689] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5722) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.434341] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=29] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-03-19 15:02:43.455263] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=34] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.173:2882", server_info_in_table={server:"10.104.106.173:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557429500476, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.456740] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=50] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.174:2882", server_info_in_table={server:"10.104.106.174:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557432223610, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.458192] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=41] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.175:2882", server_info_in_table={server:"10.104.106.175:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557431257473, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.458236] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3652) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=41] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1001}, acc_dr_task=0) [2025-03-19 15:02:43.458260] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3948) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.458280] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4515) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=18] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1001}, replicas:[{modify_time_us:1741079490915763, create_time_us:1741079485471759, tenant_id:1004, ls_id:{id:1001}, server:"10.104.106.173:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079485522580, create_time_us:1741079485465626, tenant_id:1004, ls_id:{id:1001}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079486454087, create_time_us:1741079485442023, tenant_id:1004, ls_id:{id:1001}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-03-19 15:02:43.458469] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=184] build ls locality stat map(tenant_id=1004, ls_id={id:1001}) [2025-03-19 15:02:43.458484] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.458508] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=23] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.458519] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=11] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.458539] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1741079490915763, create_time_us:1741079485471759, tenant_id:1004, ls_id:{id:1001}, server:"10.104.106.173:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079485522580, create_time_us:1741079485465626, tenant_id:1004, ls_id:{id:1001}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079486454087, create_time_us:1741079485442023, tenant_id:1004, ls_id:{id:1001}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-03-19 15:02:43.458684] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4567) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=132] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.458713] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4711) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.458732] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5433) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.458742] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5722) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.460619] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-03-19 15:02:43.481357] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=31] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.173:2882", server_info_in_table={server:"10.104.106.173:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557429500476, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.482863] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=48] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.174:2882", server_info_in_table={server:"10.104.106.174:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557432223610, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.484322] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=41] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.175:2882", server_info_in_table={server:"10.104.106.175:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557431257473, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.484366] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3652) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=40] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1002}, acc_dr_task=0) [2025-03-19 15:02:43.484397] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3948) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.484416] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4515) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1002}, replicas:[{modify_time_us:1741079486133664, create_time_us:1741079485568913, tenant_id:1004, ls_id:{id:1002}, server:"10.104.106.173:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079491091000, create_time_us:1741079485573169, tenant_id:1004, ls_id:{id:1002}, server:"10.104.106.174:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079486454087, create_time_us:1741079485543481, tenant_id:1004, ls_id:{id:1002}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-03-19 15:02:43.484604] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=183] build ls locality stat map(tenant_id=1004, ls_id={id:1002}) [2025-03-19 15:02:43.484619] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.484641] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=22] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.484653] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=10] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.484672] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1741079486133664, create_time_us:1741079485568913, tenant_id:1004, ls_id:{id:1002}, server:"10.104.106.173:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079491091000, create_time_us:1741079485573169, tenant_id:1004, ls_id:{id:1002}, server:"10.104.106.174:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079486454087, create_time_us:1741079485543481, tenant_id:1004, ls_id:{id:1002}, server:"10.104.106.175:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-03-19 15:02:43.484832] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4567) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=147] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.484861] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4711) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=18] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.484879] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5433) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.484889] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5722) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.486417] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=14] start try ls disaster recovery(ls_status_info={tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2025-03-19 15:02:43.507345] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=29] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.173:2882", server_info_in_table={server:"10.104.106.173:2882", server_id:1, zone:"zone1", sql_port:2881, with_rootserver:true, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557429500476, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.508819] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=49] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.174:2882", server_info_in_table={server:"10.104.106.174:2882", server_id:2, zone:"zone2", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557432223610, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.510268] INFO [RS] get (ob_server_table_operator.cpp:827) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=40] get server from table(ret=0, ret="OB_SUCCESS", server="10.104.106.175:2882", server_info_in_table={server:"10.104.106.175:2882", server_id:3, zone:"zone3", sql_port:2881, with_rootserver:false, status:1, build_version:"4.3.5.0_100000202024123117-5d6cb5cbc3f7c1ab6eb22e40abec8e160a8764d5(Dec 31 2024 17:35:01)", stop_time:0, start_service_time:1740557431257473, block_migrate_in_time:0, last_offline_time:0}) [2025-03-19 15:02:43.510313] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3652) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=42] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1004, ls_id={id:1003}, acc_dr_task=0) [2025-03-19 15:02:43.510337] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3948) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.510356] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4515) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] try locality alignment(dr_ls_info={resource_tenant_id_:1004, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}, {zone:"zone2", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone2"]}, {zone:"zone3", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone3"]}], inner_ls_info_:{tenant_id:1004, ls_id:{id:1003}, replicas:[{modify_time_us:1741079486134705, create_time_us:1741079485661426, tenant_id:1004, ls_id:{id:1003}, server:"10.104.106.173:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079486525857, create_time_us:1741079485656635, tenant_id:1004, ls_id:{id:1003}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1741079491112272, create_time_us:1741079485657862, tenant_id:1004, ls_id:{id:1003}, server:"10.104.106.175:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, {server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, {unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:3, schema_full_replica_cnt_:3, member_list_cnt_:3, paxos_replica_number_:3, has_leader_:true}, only_for_display=false) [2025-03-19 15:02:43.510539] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=179] build ls locality stat map(tenant_id=1004, ls_id={id:1003}) [2025-03-19 15:02:43.510554] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.510576] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=21] ls zone locality map info(zone=zone2, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.510588] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=11] ls zone locality map info(zone=zone3, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2025-03-19 15:02:43.510607] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1741079486134705, create_time_us:1741079485661426, tenant_id:1004, ls_id:{id:1003}, server:"10.104.106.173:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1005, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1005, in_pool:true, unit:unit_id:1005, resource_pool_id:1008, unit_group_id:1002, zone:"zone1", server:"10.104.106.173:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.173:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079486525857, create_time_us:1741079485656635, tenant_id:1004, ls_id:{id:1003}, server:"10.104.106.174:2882", sql_port:2881, role:2, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1006, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1006, in_pool:true, unit:unit_id:1006, resource_pool_id:1009, unit_group_id:1002, zone:"zone2", server:"10.104.106.174:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.174:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}, {replica_:{modify_time_us:1741079491112272, create_time_us:1741079485657862, tenant_id:1004, ls_id:{id:1003}, server:"10.104.106.175:2882", sql_port:2881, role:1, member_list:[{server:"10.104.106.173:2882", timestamp:1}, {server:"10.104.106.174:2882", timestamp:1}, {server:"10.104.106.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1004, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1004, in_pool:true, unit:unit_id:1004, resource_pool_id:1007, unit_group_id:1002, zone:"zone3", server:"10.104.106.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"10.104.106.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2025-03-19 15:02:43.510764] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4567) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=144] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.510785] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4711) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=11] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.510803] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5433) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.510813] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5722) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2025-03-19 15:02:43.512415] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=14] try tenant disaster recovery succ(tenant_id=1004, only_for_display=false) [2025-03-19 15:02:43.512438] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=17] start check unit migrate finish [2025-03-19 15:02:43.512454] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1) [2025-03-19 15:02:43.521322] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=16] try check migrate unit finish by tenant(tenant_id=1004) [2025-03-19 15:02:43.548221] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98029][T1_L0_G0][T1][YB420A686AAF-00062F082D89D93A-0-0] [lt=29] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2025-03-19 15:02:43.551480] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE61-0-0] [lt=30] start to check alter locality finish [2025-03-19 15:02:43.552070] INFO [RS] do_work (ob_common_ls_service.cpp:138) [785027][T1_COMMONLSSe][T1][YB420A686AAD-00062F0999ED92EC-0-0] [lt=37] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:43.552962] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE62-0-0] [lt=34][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.554418] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE62-0-0] [lt=34][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.557451] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE64-0-0] [lt=32][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.558899] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE64-0-0] [lt=31][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.560418] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE64-0-0] [lt=32] start check shrink resource pool [2025-03-19 15:02:43.560460] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE65-0-0] [lt=35] start check shrink resource pool(tenant_id=1) [2025-03-19 15:02:43.561895] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE65-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.563361] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE65-0-0] [lt=31][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.563458] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE66-0-0] [lt=31] start check shrink resource pool(tenant_id=1004) [2025-03-19 15:02:43.564894] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE66-0-0] [lt=22][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.566352] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE66-0-0] [lt=30][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.566449] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE66-0-0] [lt=31] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:43.566473] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE66-0-0] [lt=23] start check primary zone [2025-03-19 15:02:43.566490] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE67-0-0] [lt=10] start check primary zone(tenant_id=1) [2025-03-19 15:02:43.567909] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE67-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.568000] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=30] start check primary zone(tenant_id=1004) [2025-03-19 15:02:43.569426] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=21][errcode=-4008] empty result set(ret=-4008) [2025-03-19 15:02:43.569519] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=30] start do unit balance [2025-03-19 15:02:43.578874] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=21] start distribute for server status change [2025-03-19 15:02:43.578982] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=29] finish distribute for server status change(ret=0) [2025-03-19 15:02:43.579004] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=20] finish do unit balance(ret=0) [2025-03-19 15:02:43.587458] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=9] start to do tenant group unit balance [2025-03-19 15:02:43.587534] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:931) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=33] start to do tenant group unit balance(zone=zone1) [2025-03-19 15:02:43.587563] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7936) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=21] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"10.104.106.173:2882", disk_in_use_:1711276032, disk_total_:1702243598336, wild_server_:false}], over_disk_waterlevel_:false}) [2025-03-19 15:02:43.587598] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1085) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=27] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1003, 1004], group_tenant_array=[]) [2025-03-19 15:02:43.587639] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1269) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=22] generate available servers finish(ret=0, zone=zone1, available_servers=["10.104.106.173:2882"]) [2025-03-19 15:02:43.587675] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1269) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=20] generate available servers finish(ret=0, zone=zone1, available_servers=["10.104.106.173:2882"]) [2025-03-19 15:02:43.587705] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5312) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=15] start do non-tenantgroup unit balance task(zone=zone1) [2025-03-19 15:02:43.587728] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5641) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=18] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=2.279632937936137416e-01, mem_weights=1.641345894265615613e-01, log_disk_weights=6.079021167798246417e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["10.104.106.173:2882"]) [2025-03-19 15:02:43.587775] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4548) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=35] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.104.106.173:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:18, max_cpu:18, memory_size:"60GB", log_disk_size:"180GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:17, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:18, }}}, resource_info_:{cpu_capacity:80, cpu_assigned:18, cpu_assigned_max:18, mem_capacity:"370.368073GB", mem_assigned:"60GB", mem_in_use:0GB, log_disk_capacity:300GB, log_disk_assigned:180GB, log_disk_in_use:36.375GB, data_disk_capacity:1585.33789GB, data_disk_assigned:0GB, data_disk_in_use:1.59375GB}, inter_ttg_load_value_:"4.426229795567121483e-01"}], available_servers=["10.104.106.173:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.104.106.173:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:18, max_cpu:18, memory_size:"60GB", log_disk_size:"180GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:17, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:18, }}}, resource_info_:{cpu_capacity:80, cpu_assigned:18, cpu_assigned_max:18, mem_capacity:"370.368073GB", mem_assigned:"60GB", mem_in_use:0GB, log_disk_capacity:300GB, log_disk_assigned:180GB, log_disk_in_use:36.375GB, data_disk_capacity:1585.33789GB, data_disk_assigned:0GB, data_disk_in_use:1.59375GB}, inter_ttg_load_value_:"4.426229795567121483e-01"}]) [2025-03-19 15:02:43.587854] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8127) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=78] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"10.104.106.173:2882", disk_in_use_:1711276032, disk_total_:1702243598336, wild_server_:false}, over_disk_waterlevel=false) [2025-03-19 15:02:43.587876] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8133) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=22] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.104.106.173:2882", disk_in_use_:1711276032, disk_total_:1702243598336, wild_server_:false}]) [2025-03-19 15:02:43.587904] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5294) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=19] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=1, not_grant_units.count()=0) [2025-03-19 15:02:43.587931] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5349) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=24] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=4.647541285345477835e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1005, tenant_id:1004, resource_pool_id:1008, unit_config:{unit_config_id:1019, name:"config_ugs_lls_fas_zone1_S4_yqf", resource:{min_cpu:12, max_cpu:12, memory_size:"40GB", log_disk_size:"120GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:12, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:12, }}, unit:0x7eff61d5ba60, unit_config:0x7eff8444ba78, pool:0x7effeb0deb60}], not_grant_units=[], available_servers=["10.104.106.173:2882"]) [2025-03-19 15:02:43.587987] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:931) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=35] start to do tenant group unit balance(zone=zone2) [2025-03-19 15:02:43.588006] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7936) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=16] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone2", server_disk_statistic_array_:[{server_:"10.104.106.174:2882", disk_in_use_:1577058304, disk_total_:1414145245184, wild_server_:false}], over_disk_waterlevel_:false}) [2025-03-19 15:02:43.588029] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1085) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=19] generate tenant array finish(ret=0, zone=zone2, standalone_tenant_array=[1003, 1004], group_tenant_array=[]) [2025-03-19 15:02:43.588057] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1269) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=16] generate available servers finish(ret=0, zone=zone2, available_servers=["10.104.106.174:2882"]) [2025-03-19 15:02:43.588082] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1269) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=16] generate available servers finish(ret=0, zone=zone2, available_servers=["10.104.106.174:2882"]) [2025-03-19 15:02:43.588100] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5312) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=16] start do non-tenantgroup unit balance task(zone=zone2) [2025-03-19 15:02:43.588109] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5641) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=6] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone2, cpu_weights=2.279632937936137416e-01, mem_weights=1.641345894265615613e-01, log_disk_weights=6.079021167798246417e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["10.104.106.174:2882"]) [2025-03-19 15:02:43.588135] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4548) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=22] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.104.106.174:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:18, max_cpu:18, memory_size:"60GB", log_disk_size:"180GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:17, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:18, }}}, resource_info_:{cpu_capacity:80, cpu_assigned:18, cpu_assigned_max:18, mem_capacity:"370.368073GB", mem_assigned:"60GB", mem_in_use:0GB, log_disk_capacity:300GB, log_disk_assigned:180GB, log_disk_in_use:36.375GB, data_disk_capacity:1317.02539GB, data_disk_assigned:0GB, data_disk_in_use:1.46875GB}, inter_ttg_load_value_:"4.426229795567121483e-01"}], available_servers=["10.104.106.174:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.104.106.174:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:18, max_cpu:18, memory_size:"60GB", log_disk_size:"180GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:17, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:18, }}}, resource_info_:{cpu_capacity:80, cpu_assigned:18, cpu_assigned_max:18, mem_capacity:"370.368073GB", mem_assigned:"60GB", mem_in_use:0GB, log_disk_capacity:300GB, log_disk_assigned:180GB, log_disk_in_use:36.375GB, data_disk_capacity:1317.02539GB, data_disk_assigned:0GB, data_disk_in_use:1.46875GB}, inter_ttg_load_value_:"4.426229795567121483e-01"}]) [2025-03-19 15:02:43.588192] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8127) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=56] check server over disk waterlevel(zone=zone2, disk_statistic={server_:"10.104.106.174:2882", disk_in_use_:1577058304, disk_total_:1414145245184, wild_server_:false}, over_disk_waterlevel=false) [2025-03-19 15:02:43.588211] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8133) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=20] check all available servers over disk waterlevel(zone=zone2, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.104.106.174:2882", disk_in_use_:1577058304, disk_total_:1414145245184, wild_server_:false}]) [2025-03-19 15:02:43.588232] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5294) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=18] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=1, not_grant_units.count()=0) [2025-03-19 15:02:43.588253] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5349) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=20] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone2, upper_lmt=4.647541285345477835e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1006, tenant_id:1004, resource_pool_id:1009, unit_config:{unit_config_id:1020, name:"config_ugs_lls_fas_zone2_S4_pbz", resource:{min_cpu:12, max_cpu:12, memory_size:"40GB", log_disk_size:"120GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:12, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:12, }}, unit:0x7eff61d5b988, unit_config:0x7eff8444b998, pool:0x7effeb0de7a8}], not_grant_units=[], available_servers=["10.104.106.174:2882"]) [2025-03-19 15:02:43.588300] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:931) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=30] start to do tenant group unit balance(zone=zone3) [2025-03-19 15:02:43.588319] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7936) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=15] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone3", server_disk_statistic_array_:[{server_:"10.104.106.175:2882", disk_in_use_:1560281088, disk_total_:1702243598336, wild_server_:false}], over_disk_waterlevel_:false}) [2025-03-19 15:02:43.588341] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1085) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=19] generate tenant array finish(ret=0, zone=zone3, standalone_tenant_array=[1003, 1004], group_tenant_array=[]) [2025-03-19 15:02:43.588368] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1269) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=16] generate available servers finish(ret=0, zone=zone3, available_servers=["10.104.106.175:2882"]) [2025-03-19 15:02:43.588394] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1269) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=17] generate available servers finish(ret=0, zone=zone3, available_servers=["10.104.106.175:2882"]) [2025-03-19 15:02:43.588411] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5312) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=16] start do non-tenantgroup unit balance task(zone=zone3) [2025-03-19 15:02:43.588421] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5641) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=7] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone3, cpu_weights=2.279632937936137416e-01, mem_weights=1.641345894265615613e-01, log_disk_weights=6.079021167798246417e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["10.104.106.175:2882"]) [2025-03-19 15:02:43.588447] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4548) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=22] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"10.104.106.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:18, max_cpu:18, memory_size:"60GB", log_disk_size:"180GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:17, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:18, }}}, resource_info_:{cpu_capacity:80, cpu_assigned:18, cpu_assigned_max:18, mem_capacity:"370.368073GB", mem_assigned:"60GB", mem_in_use:0GB, log_disk_capacity:300GB, log_disk_assigned:180GB, log_disk_in_use:36.375GB, data_disk_capacity:1585.33789GB, data_disk_assigned:0GB, data_disk_in_use:1.453125GB}, inter_ttg_load_value_:"4.426229795567121483e-01"}], available_servers=["10.104.106.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"10.104.106.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:18, max_cpu:18, memory_size:"60GB", log_disk_size:"180GB", data_disk_size:0, min_iops:-2, max_iops:-2, iops_weight:17, max_net_bandwidth:"8.58993459e+09GB", net_bandwidth_weight:18, }}}, resource_info_:{cpu_capacity:80, cpu_assigned:18, cpu_assigned_max:18, mem_capacity:"370.368073GB", mem_assigned:"60GB", mem_in_use:0GB, log_disk_capacity:300GB, log_disk_assigned:180GB, log_disk_in_use:36.375GB, data_disk_capacity:1585.33789GB, data_disk_assigned:0GB, data_disk_in_use:1.453125GB}, inter_ttg_load_value_:"4.426229795567121483e-01"}]) [2025-03-19 15:02:43.588501] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8127) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=54] check server over disk waterlevel(zone=zone3, disk_statistic={server_:"10.104.106.175:2882", disk_in_use_:1560281088, disk_total_:1702243598336, wild_server_:false}, over_disk_waterlevel=false) [2025-03-19 15:02:43.588512] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8133) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=11] check all available servers over disk waterlevel(zone=zone3, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"10.104.106.175:2882", disk_in_use_:1560281088, disk_total_:1702243598336, wild_server_:false}]) [2025-03-19 15:02:43.588532] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5294) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=18] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=1, not_grant_units.count()=0) [2025-03-19 15:02:43.588551] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5349) [784754][RootBalance][T0][YB420A686AAD-00062F0A2226BE68-0-0] [lt=19] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone3, upper_lmt=4.647541285345477835e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1004, tenant_id:1004, resource_pool_id:1007, unit_config:{unit_config_id:1018, name:"config_ugs_lls_fas_zone3_S4_ban", resource:{min_cpu:12, max_cpu:12, memory_size:"40GB", log_disk_size:"120GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:12, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:12, }}, unit:0x7eff61d5bb38, unit_config:0x7eff8444bb58, pool:0x7effeb0def18}], not_grant_units=[], available_servers=["10.104.106.175:2882"]) [2025-03-19 15:02:43.696675] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98025][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B3B-0-0] [lt=25] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2025-03-19 15:02:43.697200] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [96661][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B3C-0-0] [lt=30] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2025-03-19 15:02:43.708130] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C8-0-0] [lt=23] is primary tenant, no need check readable_scn [2025-03-19 15:02:43.713385] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C8-0-0] [lt=33] is primary tenant, no need check readable_scn [2025-03-19 15:02:43.716526] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C8-0-0] [lt=30] is primary tenant, no need check readable_scn [2025-03-19 15:02:43.716759] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C8-0-0] [lt=29] is primary tenant, no need check readable_scn [2025-03-19 15:02:43.725271] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [784666][LeaseQueueTh2][T0][YB420A686AAE-00062F0805DA0967-0-0] [lt=33] [DDL] execute ddl like stmt(ret=0, cost=20, ddl_arg=NULL) [2025-03-19 15:02:43.746924] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98025][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B3D-0-0] [lt=45] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2025-03-19 15:02:43.769669] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [802099][T1003_StandbySc][T1003][YB420A686AAD-00062F0A7D35A98E-0-0] [lt=41] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:43.769669] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [801850][T1004_StandbySc][T1004][YB420A686AAD-00062F0A7D25A98F-0-0] [lt=41] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:43.819452] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1247) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57414-0-0] [lt=26] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-03-19 15:02:43.822771] INFO [RS] do_work (ob_common_ls_service.cpp:138) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57414-0-0] [lt=31] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:43.883291] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [785064][T1_HBService][T1][YB420A686AAD-00062F071C8EC506-0-0] [lt=38] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:44.096461] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98018][T1_L0_G0][T1][YB420A686AAF-00062F082D89D93D-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=6, ddl_arg=NULL) [2025-03-19 15:02:44.114191] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [801944][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=31] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1742367763372437, tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1742367761498578001, replayable_scn:1742367761498578001, readable_scn:1742367761498578001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}) [2025-03-19 15:02:44.155202] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [785063][T1_HBService][T1][YB420A686AAD-00062F071C9ECF98-0-0] [lt=26] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:44.283904] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [785026][T1_StandbySchem][T1][YB420A686AAD-00062F09999DA085-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:44.342745] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [784665][LeaseQueueTh1][T0][YB420A686AAD-00062F09A12D6497-0-0] [lt=36] [DDL] execute ddl like stmt(ret=0, cost=24, ddl_arg=NULL) [2025-03-19 15:02:44.374456] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98032][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B41-0-0] [lt=30] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2025-03-19 15:02:44.376823] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=16] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1742367762517046001, replayable_scn:1742367762517046001, readable_scn:1742367762517046001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1742367764373093, tenant_id=1004, tenant_info_={tenant_id:1004, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1742367762517046001, replayable_scn:1742367762517046001, readable_scn:1742367762517046001, recovery_until_scn:4611686018427387903, log_mode:"ARCHIVELOG", max_ls_id:1003, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1742367764373093, ora_rowscn_=1742367763711296000, content_changed=true) [2025-03-19 15:02:44.377235] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=48] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="10.104.106.173:2882") [2025-03-19 15:02:44.377269] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=33] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=1, dest="10.104.106.174:2882") [2025-03-19 15:02:44.377281] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [801848][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=11] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=2, dest="10.104.106.175:2882") [2025-03-19 15:02:44.387010] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [98029][T1_L0_G0][T1][YB420A686AAE-00062F0836BA6B42-0-0] [lt=35] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2025-03-19 15:02:44.412795] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958603-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1004, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1004, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1004, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476753765000, v:0}}}, {ls_id:{id:1002}, status_info:{tenant_id:1004, ls_id:{id:1002}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone2", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1002}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476766497000, v:0}}}, {ls_id:{id:1003}, status_info:{tenant_id:1004, ls_id:{id:1003}, ls_group_id:1001, status:"NORMAL", unit_group_id:1002, primary_zone:"zone3", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1003}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1741079476772834000, v:0}}}], tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:44.416101] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:468) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958603-0-0] [lt=132] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1004) [2025-03-19 15:02:44.416133] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [801853][T1004_PLSSer][T1004][YB420A686AAD-00062F0A7D958603-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1004, schema_version:1741079476931568, tenant_name:"ugs_lls_fas", zone_list:[cnt:3, "zone1", "zone2", "zone3"], primary_zone:"zone1,zone2,zone3", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1, FULL{1}@zone2, FULL{1}@zone3", zone_replica_attr_array:[cnt:3, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone2"], zone:"zone2"}, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone3"], zone:"zone3"}], primary_zone_array:[{zone_:"zone1", score_:0}, {zone_:"zone2", score_:0}, {zone_:"zone3", score_:0}], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2025-03-19 15:02:44.554020] INFO [RS] do_work (ob_common_ls_service.cpp:138) [785027][T1_COMMONLSSe][T1][YB420A686AAD-00062F0999ED92ED-0-0] [lt=31] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2025-03-19 15:02:44.647287] INFO [RS] process_ (ob_rs_rpc_processor.h:229) [784664][LeaseQueueTh0][T0][YB420A686AAF-00062F07FAA915A5-0-0] [lt=38] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2025-03-19 15:02:44.718218] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C9-0-0] [lt=19] is primary tenant, no need check readable_scn [2025-03-19 15:02:44.723589] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C9-0-0] [lt=32] is primary tenant, no need check readable_scn [2025-03-19 15:02:44.726951] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C9-0-0] [lt=27] is primary tenant, no need check readable_scn [2025-03-19 15:02:44.727152] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [801851][T1004_TeRec][T1004][YB420A686AAD-00062F0A7DD577C9-0-0] [lt=42] is primary tenant, no need check readable_scn [2025-03-19 15:02:44.769783] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [801850][T1004_StandbySc][T1004][YB420A686AAD-00062F0A7D25A990-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:44.769789] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [802099][T1003_StandbySc][T1003][YB420A686AAD-00062F0A7D35A98F-0-0] [lt=49] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2025-03-19 15:02:44.829917] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1247) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57415-0-0] [lt=27] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1", "zone2", "zone3"], unit_group_array_=[{unit_group_id:1002, unit_status:0, ls_group_ids:[1001]}]) [2025-03-19 15:02:44.833187] INFO [RS] do_work (ob_common_ls_service.cpp:138) [802100][T1003_COMMONLSS][T1003][YB420A686AAD-00062F0A7DF57415-0-0] [lt=32] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000)